# -*- coding: UTF-8 -*- ''' @Project :domainScanDemo @File :detect_worker.py @IDE :PyCharm @Author :梦伴 @Date :2026/4/10 22:40 @explain : 域名检测端程序 ''' import os import signal import sys import json import gc import math import time import socket import threading import collections import resource import schedule import urllib.error import urllib.parse import urllib.request from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from loguru import logger from queue import Empty, Full, Queue from PySide6.QtWidgets import QApplication, QMainWindow, QTextEdit, QPushButton, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QProgressBar, QGroupBox, QScrollArea, QFrame from PySide6.QtCore import Qt, QThread, Signal, QMetaObject, Q_ARG, QCoreApplication, QEvent, QTimer from PySide6.QtGui import QIcon from app.utils.database import Database from app.config import config from app.detectors.wayback_detector import WaybackDetector from app.utils.redis_client import get_redis_client from app.utils.status_codes import ( DETECT_STATUS_BLACKLISTED, DETECT_STATUS_COMPLETED, DETECT_STATUS_FAILED, DETECT_STATUS_RUNNING, REGISTER_STATUS_AVAILABLE, REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD, REVIEW_STATUS_PENDING, THIRD_PARTY_STATUS_DONE, ) import redis from detect import aizhan, baidu, c360, chinaz, register, jucha, juziseo CONFIG_UPDATE_CHANNEL = "domain_tool:config_update" CONTROL_CHANNEL = "domain_tool:worker_control" RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state" PENDING_CONTROL_KEY = "domain_tool:worker_pending_command" RUNTIME_SETTINGS_KEY = "domain_tool:runtime_settings" SHARED_PROXY_POOL_KEY_PREFIX = "domain_tool:shared_proxy_pool" SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX = "domain_tool:shared_proxy_refresh_lock" def _raise_nofile_soft_limit(): try: soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) target_limit = hard_limit if hard_limit not in (-1, resource.RLIM_INFINITY) else soft_limit if target_limit and soft_limit < target_limit: resource.setrlimit(resource.RLIMIT_NOFILE, (target_limit, hard_limit)) logger.info(f"已提升 NOFILE 软限制: {soft_limit} -> {target_limit}") except Exception as exc: logger.warning(f"提升 NOFILE 软限制失败: {exc}") def _resolve_worker_thread_stack_size_bytes() -> int: worker_mode = str(os.getenv("WORKER_MODE", "") or "").strip() raw_override = str(os.getenv("DOMAINCHECK_THREAD_STACK_SIZE_KB", "") or "").strip() if not raw_override and worker_mode != "linux-systemd": return 0 default_kb = "1024" if worker_mode == "linux-systemd" else "0" try: target_kb = int(raw_override or default_kb or 0) except Exception: target_kb = int(default_kb or 0) if target_kb <= 0: return 0 return max(256 * 1024, target_kb * 1024) def _configure_worker_thread_stack_size() -> int: target_bytes = int(_resolve_worker_thread_stack_size_bytes() or 0) try: current_bytes = int(threading.stack_size() or 0) except Exception: current_bytes = 0 if target_bytes <= 0: return current_bytes if current_bytes == target_bytes: return current_bytes try: threading.stack_size(target_bytes) applied_bytes = int(threading.stack_size() or 0) logger.info(f"已设置检测线程默认栈大小: {current_bytes} -> {applied_bytes}") return applied_bytes except Exception as exc: logger.warning(f"设置检测线程默认栈大小失败: target={target_bytes}, error={exc}") return current_bytes def _resolve_worker_log_file() -> str: candidate_dirs = [ str(getattr(config, "LOG_DIR", "") or "").strip(), os.path.join("/opt/domaincheck", "runtime", "domainCheck", "logs"), os.path.join("/tmp", "domaincheck", "logs"), ] seen = set() for raw_dir in candidate_dirs: log_dir = os.path.abspath(raw_dir) if raw_dir else "" if not log_dir or log_dir in seen: continue seen.add(log_dir) try: os.makedirs(log_dir, exist_ok=True) log_file_path = os.path.join(log_dir, "detect_worker.log") with open(log_file_path, "a", encoding="utf-8"): pass return log_file_path except Exception: continue return os.path.join("/tmp", "detect_worker.log") def _worker_local_config_roots() -> list[Path]: seen = set() roots: list[Path] = [] def append_root(raw_path) -> None: text = str(raw_path or "").strip() if not text: return path = Path(text).expanduser().resolve() key = str(path) if key in seen: return seen.add(key) roots.append(path) append_root(str(os.getenv("DOMAINCHECK_CONFIG_ROOT", "") or "").strip()) append_root(Path.cwd()) script_root = Path(__file__).resolve().parent append_root(script_root) for base_root in list(roots): normalized = str(base_root) if f"{os.sep}releases{os.sep}" in normalized: install_root = normalized.split(f"{os.sep}releases{os.sep}", 1)[0] append_root(Path(install_root) / "current" / "domainCheck") append_root(Path(install_root) / "domainCheck") elif f"{os.sep}current{os.sep}" in normalized: install_root = normalized.split(f"{os.sep}current{os.sep}", 1)[0] append_root(Path(install_root) / "current" / "domainCheck") append_root(Path(install_root) / "domainCheck") elif base_root.name == "domainCheck": append_root(base_root.parent / "current" / "domainCheck") append_root(base_root.parent / "domainCheck") return roots def _read_worker_local_json_config(filename: str): normalized_name = str(filename or "").strip() if not normalized_name: return None for root in _worker_local_config_roots(): candidate = root / normalized_name try: if candidate.exists(): with candidate.open("r", encoding="utf-8") as handle: return json.load(handle) except Exception: continue return None def pending_control_key(node_code=None): normalized_node_code = str(node_code or config.NODE_CODE or "").strip() if not normalized_node_code: return PENDING_CONTROL_KEY return f"{PENDING_CONTROL_KEY}:{normalized_node_code}" def control_target_node_codes(control_payload) -> list[str]: if not isinstance(control_payload, dict): return [] normalized_targets: list[str] = [] def append_target(raw_value): normalized_value = str(raw_value or "").strip() if normalized_value and normalized_value not in normalized_targets: normalized_targets.append(normalized_value) list_keys = ("target_node_codes", "node_codes") scalar_keys = ("target_node_code", "node_code") for key in list_keys: raw_value = control_payload.get(key) if isinstance(raw_value, (list, tuple, set)): for item in raw_value: append_target(item) elif isinstance(raw_value, str) and raw_value.strip(): for item in raw_value.split(","): append_target(item) if normalized_targets: return normalized_targets for key in scalar_keys: raw_value = control_payload.get(key) if raw_value not in (None, ""): append_target(raw_value) if normalized_targets: return normalized_targets return normalized_targets def control_targets_current_worker(control_payload, *, node_code=None) -> bool: normalized_node_code = str(node_code or config.NODE_CODE or "").strip() if not normalized_node_code: return True target_node_codes = control_target_node_codes(control_payload) if not target_node_codes: return True return normalized_node_code in target_node_codes class DetectThread(QThread): """ 检测线程 """ log_signal = Signal(str) progress_signal = Signal(int, int) finished_signal = Signal() thread_count_signal = Signal(int, int) def __init__(self, worker): super().__init__() self.worker = worker # 设置线程名称 self.setObjectName("DetectThread") def run(self): # 将信号传递给worker self.worker.detect_thread = self self.worker.start_detection() self.finished_signal.emit() class DetectMainWindow(QMainWindow): """ 检测端主窗口 """ @staticmethod def _button_style( normal_color: str, hover_color: str, pressed_color: str, disabled_color: str = "#6c757d", *, fancy_effects: bool = True, ) -> str: hover_extra = "" pressed_extra = "" disabled_extra = "" if fancy_effects: hover_extra = "\n transform: translateY(-2px);\n box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);" pressed_extra = "\n transform: translateY(1px);\n box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);" disabled_extra = "\n transform: none;\n box-shadow: none;" return f""" QPushButton {{ background-color: {normal_color}; color: white; border: none; border-radius: 8px; font-size: 16px; font-weight: bold; padding: 10px 20px; min-width: 120px; }} QPushButton:hover {{ background-color: {hover_color};{hover_extra} }} QPushButton:pressed {{ background-color: {pressed_color};{pressed_extra} }} QPushButton:disabled {{ background-color: {disabled_color}; color: #adb5bd;{disabled_extra} }} """ def __init__(self): super().__init__() self.setWindowTitle("域名检测端") self.setGeometry(100, 100, 980, 820) self.setMinimumSize(860, 680) # 设置窗口图标 # 打印当前文件路径以便诊断 current_file = os.path.abspath(__file__) current_dir = os.path.dirname(current_file) logger.info(f"当前文件路径: {current_file}") logger.info(f"当前目录: {current_dir}") # 获取PyInstaller打包后的临时目录；开发/部署态优先使用脚本所在目录。 base_dir = current_dir if hasattr(sys, '_MEIPASS'): base_dir = sys._MEIPASS logger.info(f"PyInstaller临时目录: {base_dir}") else: logger.info(f"开发环境目录: {base_dir}") def resolve_asset_path(filename: str) -> str: candidates = [ os.path.join(base_dir, filename), os.path.join(current_dir, filename), os.path.join(os.path.dirname(current_dir), filename), ] for candidate in candidates: if os.path.exists(candidate): return candidate return candidates[0] # 构建图标路径 icon_path = resolve_asset_path("new_logo.svg") logger.info(f"SVG图标路径: {icon_path}") if os.path.exists(icon_path): logger.info(f"SVG图标文件存在: {icon_path}") try: icon = QIcon(icon_path) if icon.isNull(): logger.warning(f"SVG图标加载失败，图标为空: {icon_path}") else: self.setWindowIcon(icon) logger.info(f"设置SVG窗口图标成功: {icon_path}") except Exception as e: logger.error(f"加载SVG图标时出错: {e}") else: logger.warning(f"SVG图标文件不存在: {icon_path}") # 如果SVG加载失败，使用ico文件作为备用 ico_icon_path = resolve_asset_path("favicon2.ico") logger.info(f"ICO图标路径: {ico_icon_path}") if os.path.exists(ico_icon_path): logger.info(f"ICO图标文件存在: {ico_icon_path}") try: icon = QIcon(ico_icon_path) if icon.isNull(): logger.warning(f"ICO图标加载失败，图标为空: {ico_icon_path}") else: self.setWindowIcon(icon) logger.info(f"设置ICO窗口图标成功: {ico_icon_path}") except Exception as e: logger.error(f"加载ICO图标时出错: {e}") else: logger.warning(f"ICO图标文件不存在: {ico_icon_path}") # 设置窗口样式 self.setStyleSheet("" "QMainWindow {" " background-color: #f5f5f5;" " font-family: 'Microsoft YaHei', Arial, sans-serif;" "}" "QLabel {" " font-size: 14px;" " color: #333;" " padding: 5px 0;" "}" "QGroupBox {" " font-size: 16px;" " font-weight: bold;" " border: 1px solid #ddd;" " border-radius: 8px;" " margin-top: 15px;" " padding: 15px;" " background-color: #ffffff;" "}" "QGroupBox::title {" " subcontrol-origin: margin;" " subcontrol-position: top left;" " padding: 0 15px;" " background-color: #4CAF50;" " color: white;" " border-radius: 4px;" " font-size: 14px;" "}" "QProgressBar {" " border: 1px solid #ddd;" " border-radius: 6px;" " text-align: center;" " background-color: #f0f0f0;" " height: 25px;" "}" "QProgressBar::chunk {" " background-color: #4CAF50;" " border-radius: 6px;" "}" "QTextEdit {" " font-family: Consolas, 'Courier New', monospace;" " font-size: 12px;" " border: 1px solid #ddd;" " border-radius: 6px;" " background-color: #2d2d2d;" " color: #e0e0e0;" " padding: 10px;" "}" "QPushButton {" " font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;" " font-size: 12px;" " font-weight: 600;" " padding: 6px 12px;" " border: none;" " border-radius: 4px;" " color: white;" " min-width: 80px;" " min-height: 28px;" " text-align: center;" "}" "QPushButton:hover {" " opacity: 0.9;" "}" "QPushButton:disabled {" " opacity: 0.5;" "}" "QPushButton#start_button {" " background-color: #4CAF50;" "}" "QPushButton#stop_button {" " background-color: #f44336;" "}" "QPushButton#exit_button {" " background-color: #2196F3;" "}" "") # 创建中心部件 + 内部滚动区域 scroll_area = QScrollArea() scroll_area.setWidgetResizable(True) scroll_area.setFrameShape(QFrame.NoFrame) scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.setCentralWidget(scroll_area) central_widget = QWidget() central_widget.setMinimumWidth(920) scroll_area.setWidget(central_widget) # 创建布局 layout = QVBoxLayout(central_widget) layout.setSpacing(10) layout.setContentsMargins(20, 20, 20, 20) # 创建状态标签 self.status_label = QLabel("状态: 就绪") self.status_label.setStyleSheet("font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 14px; font-weight: 600; color: #2c3e50;") layout.addWidget(self.status_label) # 创建配置信息显示区域 config_group = QGroupBox("配置信息") config_group.setStyleSheet(""" QGroupBox { font-size: 16px; font-weight: 600; color: #2c3e50; border: 1px solid #e2e8f0; border-radius: 10px; margin-top: 20px; padding: 0; background-color: #ffffff; } QGroupBox::title { subcontrol-origin: margin; subcontrol-position: top left; padding: 8px 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; border-radius: 10px 10px 0 0; font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 14px; font-weight: 600; } """) config_layout = QVBoxLayout() config_layout.setSpacing(12) config_layout.setContentsMargins(20, 20, 20, 20) # 创建配置标签 self.config_labels = { 'detect_options': QLabel("检测选项: 加载中..."), 'proxy_config': QLabel("代理配置: 加载中..."), 'thread_count': QLabel("检测线程数: 加载中...") } # 添加配置标签到布局 for label in self.config_labels.values(): label.setStyleSheet(""" QLabel { font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 13px; font-weight: 400; color: #4a5568; background-color: #f7fafc; padding: 12px 18px; border-radius: 8px; border-left: 4px solid #667eea; margin: 0; min-height: 64px; white-space: normal; border: 1px solid #e2e8f0; } QLabel:hover { background-color: #edf2f7; border-left-color: #764ba2; } """) label.setWordWrap(True) # 启用自动换行 label.setAlignment(Qt.AlignLeft | Qt.AlignTop) config_layout.addWidget(label) config_group.setLayout(config_layout) layout.addWidget(config_group) # 创建进度条 self.progress_bar = QProgressBar() self.progress_bar.setValue(0) layout.addWidget(self.progress_bar) # 创建日志文本框 self.log_text = QTextEdit() self.log_text.setReadOnly(True) self.log_text.setStyleSheet("font-family: Consolas, monospace; font-size: 12px; border: 1px solid #ddd; border-radius: 5px; background-color: #2d2d2d; color: #e0e0e0;") self.log_text.setMinimumHeight(320) layout.addWidget(self.log_text, 1) # 创建按钮布局 button_layout = QHBoxLayout() button_layout.setSpacing(10) use_fancy_button_effects = os.environ.get("QT_QPA_PLATFORM", "").lower() != "offscreen" # 按钮样式表 start_button_style = self._button_style( "#28a745", "#218838", "#1e7e34", fancy_effects=use_fancy_button_effects, ) refresh_button_style = """ QPushButton { background-color: #6f42c1; color: white; border: none; border-radius: 8px; font-size: 16px; font-weight: bold; padding: 10px 20px; min-width: 120px; } QPushButton:hover { background-color: #5a32a3; } QPushButton:disabled { background-color: #6c757d; color: #adb5bd; } """ stop_button_style = self._button_style( "#dc3545", "#c82333", "#a71e2a", fancy_effects=use_fancy_button_effects, ) exit_button_style = self._button_style( "#007bff", "#0069d9", "#0056b3", fancy_effects=use_fancy_button_effects, ) # 创建开始检测按钮 self.start_button = QPushButton("开始检测") self.start_button.setObjectName("start_button") self.start_button.clicked.connect(self.start_detection) self.start_button.setFixedHeight(50) self.start_button.setStyleSheet(start_button_style) button_layout.addWidget(self.start_button) # 创建停止按钮 self.stop_button = QPushButton("停止检测") self.stop_button.setObjectName("stop_button") self.stop_button.clicked.connect(self.stop_detection) self.stop_button.setEnabled(False) self.stop_button.setFixedHeight(50) self.stop_button.setStyleSheet(stop_button_style) button_layout.addWidget(self.stop_button) self.refresh_proxy_button = QPushButton("刷新代理池") self.refresh_proxy_button.clicked.connect(self.refresh_proxy_pool_manually) self.refresh_proxy_button.setFixedHeight(50) self.refresh_proxy_button.setStyleSheet(refresh_button_style) button_layout.addWidget(self.refresh_proxy_button) # 创建退出按钮 self.exit_button = QPushButton("退出") self.exit_button.setObjectName("exit_button") self.exit_button.clicked.connect(self.close) self.exit_button.setFixedHeight(50) self.exit_button.setStyleSheet(exit_button_style) button_layout.addWidget(self.exit_button) layout.addLayout(button_layout) # 初始化检测端 self.worker = DetectWorker(self) # 激活服务态运行时：启动控制链路，并在服务重启后立即尝试回挂活动检测任务。 self.worker.activate_service_runtime() # 连接信号 self.worker.connect_signals() # 初始化检测线程 self.detect_thread = None # 初始化日志队列和处理线程 self.log_queue = Queue() self.log_processing = True # 启动日志处理线程 self.log_thread = threading.Thread(target=self.process_log_queue) self.log_thread.daemon = True self.log_thread.start() # 配置日志 logger.add(self.enqueue_log_message, level="INFO") # 启动定时任务 self.scheduler_thread = threading.Thread(target=self.start_scheduler) self.scheduler_thread.daemon = True self.scheduler_thread.start() def enqueue_log_message(self, message): """ 日志回调函数，将日志消息放入队列 """ self.log_queue.put(message) def process_log_queue(self): """ 处理日志队列中的消息 """ while self.log_processing: try: message = self.log_queue.get(timeout=1) # 提取时间和消息部分 # 日志格式: 2026-04-13 00:22:05.365 | INFO | __main__:load_thread_count:846 - 从Redis加载检测线程数成功: 500 import re match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| .*? \| .*? - (.*)', message) if match: time_str = match.group(1) msg_str = match.group(2) log_text = f"{time_str} | {msg_str}" else: log_text = message.replace("\n", "") # 使用QMetaObject.invokeMethod确保线程安全的GUI更新 QMetaObject.invokeMethod(self.log_text, "append", Qt.QueuedConnection, Q_ARG(str, log_text)) QMetaObject.invokeMethod(self.log_text.verticalScrollBar(), "setValue", Qt.QueuedConnection, Q_ARG(int, self.log_text.verticalScrollBar().maximum())) except Exception as e: pass def log_message(self, message): """ 日志回调函数（保留用于信号连接） """ self.enqueue_log_message(message) def start_detection(self): """ 开始检测 """ self.status_label.setText("状态: 检测中...") self.start_button.setEnabled(False) self.stop_button.setEnabled(True) # 创建并启动检测线程 self.detect_thread = DetectThread(self.worker) self.detect_thread.log_signal.connect(self.log_message) self.detect_thread.progress_signal.connect(self.update_progress) self.detect_thread.thread_count_signal.connect(self.update_thread_count_display) self.detect_thread.finished_signal.connect(self.detection_finished) self.detect_thread.start() def stop_detection(self): """ 停止检测 """ self.status_label.setText("状态: 停止中...") self.worker.stop() self.stop_button.setEnabled(False) def detection_finished(self): """ 检测完成 """ self.status_label.setText("状态: 检测完成") self.start_button.setEnabled(True) self.stop_button.setEnabled(False) def update_progress(self, current, total): """ 更新进度条 """ # 直接在当前线程执行，避免使用QMetaObject.invokeMethod try: if total > 0: progress = int((current / total) * 100) self.progress_bar.setValue(progress) except Exception as e: print(f"更新进度失败: {e}") def update_thread_count_display(self, active_count, total_count): """ 更新线程数量显示 :param active_count: 当前活跃线程数量 :param total_count: 总线程数量 """ try: # 检查是否有配置标签 if hasattr(self, 'config_labels') and 'thread_count' in self.config_labels: self.config_labels['thread_count'].setText(f"检测线程数: {total_count} (当前: {active_count})") except Exception as e: print(f"更新线程数量显示失败: {e}") def refresh_proxy_pool_manually(self): self.status_label.setText("状态: 正在刷新代理池...") self.refresh_proxy_button.setEnabled(False) def run_refresh(): try: self.worker.refresh_proxy_pool() with self.worker.proxy_pool_lock: proxy_count = len(self.worker.proxy_pool) self.worker.update_config_labels() QMetaObject.invokeMethod( self.status_label, "setText", Qt.QueuedConnection, Q_ARG(str, f"状态: 代理池刷新完成，可用代理 {proxy_count} 个") ) except Exception as e: QMetaObject.invokeMethod( self.status_label, "setText", Qt.QueuedConnection, Q_ARG(str, f"状态: 刷新代理池失败: {e}") ) finally: QMetaObject.invokeMethod( self.refresh_proxy_button, "setEnabled", Qt.QueuedConnection, Q_ARG(bool, True) ) threading.Thread(target=run_refresh, daemon=True).start() def start_scheduler(self): """ 启动定时任务 """ # 每天凌晨1点执行检测 schedule.every().day.at("01:00").do(self.scheduled_detection) # 循环执行定时任务 while True: schedule.run_pending() time.sleep(60) def scheduled_detection(self): """ 定时检测任务 """ self.log_message(f"[{datetime.now()}] 开始定时检测任务") self.start_detection() def refresh_config(self): """ 刷新配置 """ try: # 重新加载配置 new_detect_options = self.worker.load_detect_options() new_proxy_config = self.worker.load_proxy_config() new_thread_count = self.worker.load_thread_count() new_sensitive_words = self.worker.db.get_all_sensitive_words() self.worker.load_cookies_from_remote() # 检查配置是否发生变化 config_changed = False if new_detect_options != self.worker.detect_options: self.worker.detect_options = new_detect_options config_changed = True if new_proxy_config != self.worker.proxy_config: self.worker.proxy_config = new_proxy_config config_changed = True if new_thread_count != self.worker.thread_count: self.worker.thread_count = new_thread_count config_changed = True if new_sensitive_words != self.worker.sensitive_words: self.worker.sensitive_words = new_sensitive_words config_changed = True # 如果配置发生变化，更新标签 if config_changed: self.worker.update_config_labels() self.log_message(f"[{datetime.now()}] 配置已更新") except Exception as e: self.log_message(f"[{datetime.now()}] 刷新配置失败: {e}") def event(self, event): """ 处理事件 """ if event.type() == ConfigUpdateEvent.Type: self.update_config_labels() return True return super().event(event) def update_config_labels(self): """ 更新GUI配置标签 """ logger.debug("开始更新GUI配置标签") try: if self.worker: try: logger.debug(f"worker对象存在，当前配置: {self.worker.detect_options}, {self.worker.proxy_config}, {self.worker.thread_count}") # 准备更新数据 detect_options = self.worker.detect_options free_options_display = [] paid_options_display = [] option_names = { 'detect_register': '检查注册', 'detect_baidu_site': '百度site查询', 'detect_360_site': '360的site查询', 'detect_chinaz': '站长之家查询', 'detect_aizhan': '爱站网查询', 'detect_wayback': '时光机检测', 'detect_jucha': '聚查查询', 'detect_juziseo': '桔子查询' } ordered_keys = detect_options.get('detect_order') or list(option_names.keys()) normalized_keys = [key for key in ordered_keys if key in option_names] for key in option_names: if key not in normalized_keys: normalized_keys.append(key) free_keys = { 'detect_register', 'detect_baidu_site', 'detect_360_site', 'detect_chinaz', 'detect_aizhan', 'detect_wayback', } for index, key in enumerate(normalized_keys, start=1): name = option_names[key] if key in detect_options and detect_options[key]: item_text = f"{index}.{name}" if key in free_keys: free_options_display.append(item_text) else: paid_options_display.append(item_text) free_text = " | ".join(free_options_display) if free_options_display else "无" paid_text = " | ".join(paid_options_display) if paid_options_display else "无" detect_options_text = ( "当前检测选项:\n" f"免费优先: {free_text}\n" f"付费后置: {paid_text}" ) # 准备代理配置文本 proxy_enable = self.worker.proxy_config.get('proxy_enable', False) allow_direct = self.worker.proxy_config.get('allow_direct', False) proxy_urls = self.worker.proxy_config.get('proxy_urls') or [] if not proxy_urls and self.worker.proxy_config.get('proxy_url'): proxy_urls = [self.worker.proxy_config.get('proxy_url', '')] with self.worker.proxy_pool_lock: proxy_pool_count = len(self.worker.proxy_pool) last_refresh_time = ( self.worker.proxy_last_refresh_time.strftime('%Y-%m-%d %H:%M:%S') if self.worker.proxy_last_refresh_time else '未刷新' ) proxy_display = ( f"启用: {'是' if proxy_enable else '否'} | " f"允许直连: {'是' if allow_direct else '否'} | " f"代理池链接数: {len([url for url in proxy_urls if url])} | " f"当前可用代理数: {proxy_pool_count}\n" f"最近刷新: {last_refresh_time} | " f"最近结果: {self.worker.proxy_last_refresh_status} | " f"原始返回数: {self.worker.proxy_last_refresh_total_items}" ) proxy_text = f"代理配置: {proxy_display}" # 准备线程数文本 thread_count_text = f"检测线程数: {self.worker.thread_count}" logger.debug(f"准备更新标签: {detect_options_text}, {proxy_text}, {thread_count_text}") # 检查配置标签是否存在 if hasattr(self, 'config_labels'): # 使用QMetaObject.invokeMethod确保在主线程中更新UI from PySide6.QtCore import QMetaObject, Qt, Q_ARG # 更新检测选项标签 if 'detect_options' in self.config_labels: QMetaObject.invokeMethod(self.config_labels['detect_options'], "setText", Qt.QueuedConnection, Q_ARG(str, detect_options_text)) # 更新代理配置标签 if 'proxy_config' in self.config_labels: QMetaObject.invokeMethod(self.config_labels['proxy_config'], "setText", Qt.QueuedConnection, Q_ARG(str, proxy_text)) # 更新线程数标签 if 'thread_count' in self.config_labels: QMetaObject.invokeMethod(self.config_labels['thread_count'], "setText", Qt.QueuedConnection, Q_ARG(str, thread_count_text)) logger.debug("GUI配置标签更新完成") else: logger.debug("配置标签不存在，跳过更新") except Exception as e: logger.error(f"更新GUI标签失败: {e}") import traceback logger.error(traceback.format_exc()) else: logger.debug("worker对象不存在") except Exception as e: logger.error(f"执行update_config_labels失败: {e}") import traceback logger.error(traceback.format_exc()) from PySide6.QtCore import QObject, Signal, QEvent class ConfigUpdateEvent(QEvent): """ 配置更新事件类 """ Type = QEvent.Type(QEvent.User + 1) def __init__(self): super().__init__(ConfigUpdateEvent.Type) class ConfigUpdateSignal(QObject): """ 配置更新信号类 """ config_updated = Signal() def runtime_state_key(node_code=None): normalized_node_code = str(node_code or config.NODE_CODE or "").strip() if not normalized_node_code: return RUNTIME_STATE_KEY return f"{RUNTIME_STATE_KEY}:{normalized_node_code}" class DetectWorker: """ 域名检测端工作类 """ def __init__(self, main_window=None): """ 初始化检测端 :param main_window: 主窗口实例，用于更新GUI """ self.main_window = main_window self.running = False self.stop_requested = False self.detecting = False self.detect_threads = [] self.detect_thread = None # 检测线程实例 self.detect_command_thread = None self.detect_lock = threading.Lock() self._domain_thread_counter_lock = threading.Lock() self._task_local = threading.local() self._step_result_cache_lock = threading.Lock() self._step_result_cache = {} self._active_domain_threads = 0 self.detect_command_started_at = 0.0 self.detect_command_last_activity_at = 0.0 self._last_domain_started_at = 0.0 self._last_domain_result_at = 0.0 self._pending_restart_source = "" self._pending_restart_payload = None self._pending_restart_reason = "" self._detect_session_seq = 0 self._detect_session_owner = 0 self._worker_started_at = time.time() self._last_worker_activity_at = self._worker_started_at self._last_explicit_start_signal_at = 0.0 self._last_explicit_start_payload = {} self._last_detection_start_at = 0.0 self._last_detection_start_source = "" self._last_detection_start_job_id = None self._last_detection_start_job_code = "" self._last_detection_start_task_mode = "" self._cached_active_detect_job = None self._last_active_detect_job_refresh_at = 0.0 self._explicit_claim_scope_job_id = None self._explicit_claim_scope_job_code = "" self._explicit_claim_scope_until = 0.0 self._explicit_claim_scope_source = "" self._ignored_targeted_job_id = None self._ignored_targeted_job_code = "" self._ignored_targeted_job_until = 0.0 self._ignored_targeted_job_targets = "" self._last_explicit_scope_claim_log_at = 0.0 self._last_explicit_scope_claim_log_key = "" self._last_explicit_scope_probe_log_at = 0.0 self._last_job_status_refresh_probe_log_at = 0.0 self._idle_recycle_requested = False self._last_thread_count_refresh_at = 0.0 self.runtime_heartbeat_interval = max( 3.0, float(os.getenv("DOMAINCHECK_RUNTIME_HEARTBEAT_INTERVAL", "5") or 5), ) self._runtime_heartbeat_stop = threading.Event() self._last_runtime_phase = "idle" self._last_runtime_detail = "Worker 初始化中" self._last_runtime_extra = {} self._last_runtime_state_push_at = 0.0 self._last_runtime_state_push_phase = "" self.current_cycle_token = "" self.current_job_id = None self.current_job_code = "" self.current_job_task_mode = "" self._restart_release_handoff_job_id = None self._restart_release_handoff_job_code = "" self._restart_release_handoff_until = 0.0 self._restart_release_handoff_reason = "" self.runtime_settings = { "worker_log_sync_enabled": False, "worker_log_sync_mode": "key", "worker_step_trace_enabled": True, "worker_step_trace_sync_full": True, "claim_batch_floor": 0, "claim_batch_ceil": 0, "submit_backlog_floor": 0, "submit_backlog_ceil": 0, "dispatch_cap_multiplier": 1, "pending_buffer_cap_multiplier": 1, } self._last_synced_worker_log = "" self._worker_log_sync_queue = Queue(maxsize=50000) self._worker_log_sync_stop = threading.Event() self._worker_log_sync_drop_count = 0 self._last_worker_log_sync_drop_notice_at = 0.0 self._last_worker_log_sync_degrade_notice_at = 0.0 self._remote_debug_event_failure_streak = 0 self._remote_debug_event_cooldown_until = 0.0 self._last_remote_debug_event_notice_at = 0.0 self._running_mark_lock = threading.Lock() self._pending_running_marks = collections.deque() self._last_running_mark_flush_at = 0.0 self._job_release_lock = threading.Lock() self._pending_job_releases = collections.deque() self._pending_job_release_reasons = {} self._last_job_release_flush_at = 0.0 self._job_finalize_lock = threading.Lock() self._pending_job_finalizations = collections.deque() self._last_job_finalize_flush_at = 0.0 self._domain_status_update_lock = threading.Lock() self._pending_domain_status_updates = collections.deque() self._last_domain_status_flush_at = 0.0 self._domain_completion_lock = threading.Lock() self._pending_domain_completions = collections.deque() self._last_domain_completion_flush_at = 0.0 self._review_status_update_lock = threading.Lock() self._pending_review_status_updates = collections.deque() self._last_review_status_flush_at = 0.0 self._completed_future_lock = threading.Lock() self._completed_futures = collections.deque() self._last_job_item_release_at = 0.0 # 初始化数据库连接 self.db = Database() try: self.db.ensure_cluster_runtime_tables() self.db.register_cluster_node( config.NODE_CODE, config.NODE_REGION, config.NODE_ROLE, status="online", current_load=0, metadata={"service": "detect-worker", "worker_mode": os.environ.get("WORKER_MODE", "")}, ) except Exception as e: logger.warning(f"初始化多机运行库骨架失败: {e}") try: recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED) if recycled: logger.warning(f"Worker 启动时回收了 {recycled} 个遗留的检测中域名，已标记为失败待重试") released = self._maybe_release_node_job_items(reason="worker_bootstrap", min_interval_seconds=0.0) if released: logger.warning(f"Worker 启动时释放了 {released} 个当前节点遗留任务项，已重新回到 pending") except Exception as e: logger.warning(f"Worker 启动时清理遗留运行态失败: {e}") # 初始化Redis连接 try: self.redis_client = get_redis_client(role="standard") # 测试连接 self.redis_client.ping() logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}") self.use_redis = True except Exception as e: logger.warning(f"Redis 连接失败: {e}，将使用本地配置") self.redis_client = None self.use_redis = False # 初始化配置更新信号 self.config_signal = ConfigUpdateSignal() # 加载配置 self.detect_options = self.load_detect_options() self.proxy_config = self.load_proxy_config() self.thread_count = self.load_thread_count() # 从配置文件加载线程数 self.process_count = self.load_process_count() # 从配置文件加载进程数 self.runtime_settings = self.load_runtime_settings() self.register_single_machine_mode_enabled = self._env_flag_enabled( "DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE" ) self.single_machine_site_direct_fallback_enabled = self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK", default=self.register_single_machine_mode_enabled, ) self.single_machine_aizhan_direct_first_enabled = self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST", default=False, ) self.single_machine_baidu_direct_first_enabled = self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST", default=False, ) self.aizhan_remote_disconnect_degrade_enabled = self._env_flag_enabled( "DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE", default=False, ) self.aizhan_external_fast_degrade_enabled = self._env_flag_enabled( "DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE", default=False, ) self.register_single_machine_direct_streak_attempts = max( 0, int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2), ) if self.register_single_machine_mode_enabled else 0 # 初始化代理池 self.proxy_pool = [] self.proxy_pool_lock = threading.Lock() self.proxy_refresh_lock = threading.Lock() self.proxy_last_refresh_time = None self.proxy_last_refresh_status = "未刷新" self.proxy_last_refresh_source_count = 0 self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = 0 self.proxy_last_source_stats = [] self._last_proxy_clock_skew_warning_at = 0.0 self.proxy_refresh_cooldown_seconds = 30 self.proxy_next_refresh_time = 0.0 self._last_proxy_refresh_reason_at = {} self.proxy_max_reuse_count = max(8, self.thread_count * 4) self.proxy_step_wait_timeout_seconds = max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_WAIT_TIMEOUT", "1.5") or 1.5), ) self.proxy_direct_fallback_grace_seconds = max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE", "0.35") or 0.35), ) self.proxy_direct_fallback_grace_overrides = { "注册状态检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_REGISTER", "0.08") or 0.08), ), "360检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_360", "0.12") or 0.12), ), "百度site检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_BAIDU", "0.12") or 0.12), ), "站长之家检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_CHINAZ", "0.12") or 0.12), ), "爱站网检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_AIZHAN", "0.12") or 0.12), ), "时光机检测": max( 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_WAYBACK", "0.2") or 0.2), ), } self.proxy_direct_retry_wait_seconds = max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT", "1.2") or 1.2), ) self.proxy_direct_retry_wait_overrides = { "注册状态检测": max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_REGISTER", "0.2") or 0.2), ), "360检测": max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_360", "0.25") or 0.25), ), "百度site检测": max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_BAIDU", "0.25") or 0.25), ), "站长之家检测": max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_CHINAZ", "0.25") or 0.25), ), "爱站网检测": max( 0.0, float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_AIZHAN", "0.25") or 0.25), ), } self.proxy_step_retry_max_attempts = max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS", "12") or 12), ) self.proxy_step_retry_max_seconds = max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS", "45") or 45), ) self.proxy_step_retry_budget_overrides = { "注册状态检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_REGISTER", "3") or 3), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_REGISTER", "8") or 8), ), }, "百度site检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_BAIDU", "3") or 3), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_BAIDU", "6") or 6), ), }, "360检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_360", "2") or 2), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_360", "5") or 5), ), }, "站长之家检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_CHINAZ", "2") or 2), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_CHINAZ", "5") or 5), ), }, "爱站网检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_AIZHAN", "2") or 2), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_AIZHAN", "5") or 5), ), }, "时光机检测": { "max_attempts": max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_WAYBACK", "3") or 3), ), "max_seconds": max( 5.0, float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_WAYBACK", "12") or 12), ), }, } self._last_no_proxy_notice_at = 0.0 self.proxy_failure_lock = threading.Lock() self.proxy_failure_counts = {} self.proxy_quarantine_until = {} self.proxy_source_by_key = {} self.proxy_source_failure_counts = {} self.proxy_source_quarantine_until = {} self.proxy_active_leases = {} self.redis_sub_thread = None self._last_autoresume_skip_log_at = 0.0 self._last_autoresume_success_log_at = 0.0 # 加载敏感词（只加载一次，所有线程共享） try: self.sensitive_words = self._load_sensitive_words_runtime() logger.info(f"加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词") except Exception as e: logger.error(f"加载敏感词失败: {e}") import traceback logger.error(traceback.format_exc()) self.sensitive_words = [] # 初始化运行状态 self.running = False # 加载cookies self.load_cookies_from_remote() # 付费检测器按需初始化，避免在默认关闭时产生无效启动告警。 self.jc_instance = None self.juziseo_instance = None self._init_optional_detectors() self.wayback_detector = WaybackDetector() # 显示所有配置信息 logger.info("检测端配置信息:") logger.info(f"检测选项: {self.detect_options}") logger.info(f"代理配置: {self.proxy_config}") logger.info(f"检测线程数: {self.thread_count}") logger.info(f"检测进程数: {self.process_count}") # Redis订阅线程将在start方法中启动 self._update_runtime_state("idle", "Worker 已启动，等待检测指令") self._start_runtime_heartbeat_loop() self._start_worker_log_sync_loop() self.trigger_proxy_refresh(reason="worker_startup", reset_cooldown=True) def _start_runtime_heartbeat_loop(self): existing = getattr(self, "_runtime_heartbeat_thread", None) if existing and existing.is_alive(): return self._runtime_heartbeat_stop.clear() self._runtime_heartbeat_thread = threading.Thread( target=self._runtime_heartbeat_loop, name="RuntimeHeartbeatThread", daemon=True, ) self._runtime_heartbeat_thread.start() def _runtime_heartbeat_loop(self): while not self._runtime_heartbeat_stop.wait(self.runtime_heartbeat_interval): try: self._consume_pending_control_command() except Exception as e: logger.debug(f"运行态心跳补偿消费待执行控制指令失败: {e}") try: self._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") except Exception as e: logger.debug(f"运行态心跳自动回挂检测任务失败: {e}") try: self._update_runtime_state( self._last_runtime_phase, self._last_runtime_detail, **dict(self._last_runtime_extra or {}), ) except Exception as e: logger.debug(f"运行态心跳续期失败: {e}") try: self._maybe_recycle_idle_worker(trigger_reason="heartbeat") except Exception as e: logger.debug(f"空闲 Worker 生命周期检查失败: {e}") def _start_worker_log_sync_loop(self): existing = getattr(self, "_worker_log_sync_thread", None) if existing and existing.is_alive(): return self._worker_log_sync_stop.clear() self._worker_log_sync_thread = threading.Thread( target=self._worker_log_sync_loop, name="WorkerLogSyncThread", daemon=True, ) self._worker_log_sync_thread.start() def _worker_log_sync_loop(self): while True: if self._worker_log_sync_stop.is_set() and self._worker_log_sync_queue.empty(): break try: item = self._worker_log_sync_queue.get(timeout=0.5) except Empty: continue if item is None: self._worker_log_sync_queue.task_done() if self._worker_log_sync_stop.is_set(): break continue if item.get("write_db", True): try: self.db.append_detect_run_event( item["job_id"], item["job_item_id"], config.NODE_CODE, event_type='worker_log', message=item["message"], level=item["level"], payload=item["payload"], ) except Exception as e: logger.debug(f"Worker 日志事件写库失败: {e}") try: self._push_remote_debug_event( message=item["message"], level=item["level"], payload=item["payload"], event_type="worker_log", ) except Exception as e: logger.debug(f"Worker 日志事件远端回传失败: {e}") finally: self._worker_log_sync_queue.task_done() def _enqueue_running_mark(self, job_item_id, claim_token): normalized_claim_token = str(claim_token or "").strip() try: normalized_job_item_id = int(job_item_id or 0) except Exception: normalized_job_item_id = 0 if normalized_job_item_id <= 0 or not normalized_claim_token: return with self._running_mark_lock: self._pending_running_marks.append((normalized_job_item_id, normalized_claim_token)) def _flush_pending_running_marks(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_running_mark_flush_at", 0.0) or 0.0) < 0.05: return 0 batch = [] with self._running_mark_lock: if not self._pending_running_marks: if force: self._last_running_mark_flush_at = now_ts return 0 while self._pending_running_marks and len(batch) < max(1, int(batch_limit or 1)): batch.append(self._pending_running_marks.popleft()) updated_count = 0 try: updated_count = int(self.db.mark_detect_job_items_running_batch(batch) or 0) except Exception as e: logger.debug(f"批量刷新运行中任务项失败: {e}") updated_count = -1 if updated_count < 0: with self._running_mark_lock: for item in reversed(batch): self._pending_running_marks.appendleft(item) logger.warning( f"批量刷新运行中任务项失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 self._last_running_mark_flush_at = now_ts return updated_count def _enqueue_job_finalization( self, *, job_item_id, claim_token, final_status, message="", result_payload=None, result_version="v1", job_id=None, node_code="", event_type="", event_level="info", event_message="", event_payload=None, ): normalized_claim_token = str(claim_token or "").strip() try: normalized_job_item_id = int(job_item_id or 0) except Exception: normalized_job_item_id = 0 if normalized_job_item_id <= 0 or not normalized_claim_token: return try: normalized_job_id = int(job_id or 0) except Exception: normalized_job_id = 0 with self._job_finalize_lock: self._pending_job_finalizations.append( { "job_item_id": normalized_job_item_id, "claim_token": normalized_claim_token, "final_status": str(final_status or "").strip() or "failed", "message": str(message or "").strip(), "result_payload": result_payload, "result_version": str(result_version or "v1").strip() or "v1", "job_id": normalized_job_id, "node_code": str(node_code or "").strip(), "event_type": str(event_type or "").strip(), "event_level": str(event_level or "info").strip() or "info", "event_message": str(event_message or "").strip(), "event_payload": dict(event_payload or {}), } ) def _enqueue_job_release(self, job_item_id, claim_token, reason=""): normalized_claim_token = str(claim_token or "").strip() try: normalized_job_item_id = int(job_item_id or 0) except Exception: normalized_job_item_id = 0 if normalized_job_item_id <= 0 or not normalized_claim_token: return False release_key = (normalized_job_item_id, normalized_claim_token) normalized_reason = str(reason or "").strip()[:1000] with self._job_release_lock: existing_reason = str( getattr(self, "_pending_job_release_reasons", {}).get(release_key, "") or "" ).strip() if release_key not in getattr(self, "_pending_job_release_reasons", {}): self._pending_job_releases.append(release_key) # Keep the newest non-empty reason, but avoid growing the queue for duplicates. self._pending_job_release_reasons[release_key] = normalized_reason or existing_reason return True def _flush_pending_job_releases(self, *, force=False, batch_limit=128): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_job_release_flush_at", 0.0) or 0.0) < 0.20: return 0 batch = [] with self._job_release_lock: if not self._pending_job_releases: if force: self._last_job_release_flush_at = now_ts return 0 while self._pending_job_releases and len(batch) < max(1, int(batch_limit or 1)): release_key = self._pending_job_releases.popleft() reason = str(self._pending_job_release_reasons.pop(release_key, "") or "").strip() batch.append((int(release_key[0]), str(release_key[1]), reason)) updated_count = 0 flush_started_at = time.perf_counter() try: updated_count = int(self.db.release_detect_job_items_batch(batch) or 0) except Exception as e: logger.debug(f"批量释放任务项回队列失败: {e}") updated_count = -1 if updated_count < 0: with self._job_release_lock: for item in reversed(batch): release_key = (int(item[0]), str(item[1])) self._pending_job_releases.appendleft(release_key) self._pending_job_release_reasons[release_key] = str(item[2] or "").strip()[:1000] logger.warning( f"批量释放任务项回队列失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000) if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force): logger.info( f"任务项回队列批量释放: batch={len(batch)} | updated={updated_count} " f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}" ) self._last_job_release_flush_at = now_ts return updated_count def _flush_pending_job_finalizations(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_job_finalize_flush_at", 0.0) or 0.0) < 0.05: return 0 batch = [] with self._job_finalize_lock: if not self._pending_job_finalizations: if force: self._last_job_finalize_flush_at = now_ts return 0 while self._pending_job_finalizations and len(batch) < max(1, int(batch_limit or 1)): batch.append(self._pending_job_finalizations.popleft()) updated_count = 0 flush_started_at = time.perf_counter() try: updated_count = int(self.db.finalize_detect_job_items_batch(batch) or 0) except Exception as e: logger.debug(f"批量回写任务项完成态失败: {e}") updated_count = -1 if updated_count < 0: with self._job_finalize_lock: for item in reversed(batch): self._pending_job_finalizations.appendleft(item) logger.warning( f"批量回写任务项完成态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000) if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force): logger.info( f"任务项完成态批量回写: batch={len(batch)} | updated={updated_count} " f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}" ) self._last_job_finalize_flush_at = now_ts return updated_count def _enqueue_domain_status_update(self, domain_id, status): try: normalized_domain_id = int(domain_id or 0) normalized_status = int(status) except Exception: return if normalized_domain_id <= 0: return with self._domain_status_update_lock: self._pending_domain_status_updates.append((normalized_domain_id, normalized_status)) def _enqueue_domain_completion(self, domain_id, *, register_status, use_status, expire_date): try: normalized_domain_id = int(domain_id or 0) normalized_register_status = int(register_status or 0) normalized_use_status = int(use_status or 0) except Exception: return if normalized_domain_id <= 0: return with self._domain_completion_lock: self._pending_domain_completions.append( ( normalized_domain_id, normalized_register_status, normalized_use_status, bool(expire_date), ) ) def _flush_pending_domain_completions(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_domain_completion_flush_at", 0.0) or 0.0) < 0.05: return 0 batch = [] with self._domain_completion_lock: if not self._pending_domain_completions: if force: self._last_domain_completion_flush_at = now_ts return 0 while self._pending_domain_completions and len(batch) < max(1, int(batch_limit or 1)): batch.append(self._pending_domain_completions.popleft()) updated_count = 0 flush_started_at = time.perf_counter() try: updated_count = int(self.db.complete_domain_detection_batch(batch) or 0) except Exception as e: logger.debug(f"批量刷新域名完成态失败: {e}") updated_count = -1 if updated_count < 0: with self._domain_completion_lock: for item in reversed(batch): self._pending_domain_completions.appendleft(item) logger.warning( f"批量刷新域名完成态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000) if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force): logger.info( f"域名完成态批量回写: batch={len(batch)} | updated={updated_count} " f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}" ) self._last_domain_completion_flush_at = now_ts return updated_count def _flush_pending_domain_status_updates(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_domain_status_flush_at", 0.0) or 0.0) < 0.05: return 0 batch = [] with self._domain_status_update_lock: if not self._pending_domain_status_updates: if force: self._last_domain_status_flush_at = now_ts return 0 while self._pending_domain_status_updates and len(batch) < max(1, int(batch_limit or 1)): batch.append(self._pending_domain_status_updates.popleft()) updated_count = 0 try: updated_count = int(self.db.update_domain_detect_status_batch(batch) or 0) except Exception as e: logger.debug(f"批量刷新域名检测状态失败: {e}") updated_count = -1 if updated_count < 0: with self._domain_status_update_lock: for item in reversed(batch): self._pending_domain_status_updates.appendleft(item) logger.warning( f"批量刷新域名检测状态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 self._last_domain_status_flush_at = now_ts return updated_count def _enqueue_review_status_update(self, domain_id, review_status): try: normalized_domain_id = int(domain_id or 0) normalized_review_status = int(review_status) except Exception: return if normalized_domain_id <= 0: return with self._review_status_update_lock: self._pending_review_status_updates.append((normalized_domain_id, normalized_review_status)) def _flush_pending_review_status_updates(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_review_status_flush_at", 0.0) or 0.0) < 0.05: return 0 batch = [] with self._review_status_update_lock: if not self._pending_review_status_updates: if force: self._last_review_status_flush_at = now_ts return 0 while self._pending_review_status_updates and len(batch) < max(1, int(batch_limit or 1)): batch.append(self._pending_review_status_updates.popleft()) updated_count = 0 try: updated_count = int(self.db.update_domain_review_status_batch(batch) or 0) except Exception as e: logger.debug(f"批量刷新域名复核状态失败: {e}") updated_count = -1 if updated_count < 0: with self._review_status_update_lock: for item in reversed(batch): self._pending_review_status_updates.appendleft(item) logger.warning( f"批量刷新域名复核状态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" ) updated_count = 0 self._last_review_status_flush_at = now_ts return updated_count def _enqueue_completed_future(self, future): if future is None: return with self._completed_future_lock: self._completed_futures.append(future) def _consume_pending_control_command(self): if not self.use_redis or self.redis_client is None: return pending_keys = [pending_control_key()] if pending_keys[0] != PENDING_CONTROL_KEY: pending_keys.append(PENDING_CONTROL_KEY) payload = None payload_key = "" try: for key in pending_keys: payload = self.redis_client.getdel(key) if payload: payload_key = key break except Exception: try: for key in pending_keys: payload = self.redis_client.get(key) if payload: self.redis_client.delete(key) payload_key = key break except Exception as e: logger.debug(f"读取待执行控制指令失败: {e}") return if not payload: return logger.info(f"发现待执行 Worker 控制指令: key={payload_key or 'unknown'} payload={payload}") self._handle_control_message(payload) def _active_detect_job_refresh_interval_seconds(self, reason: str) -> float: if reason == "heartbeat_autoresume": raw_value = os.getenv("DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS", "5") else: raw_value = os.getenv("DOMAINCHECK_ACTIVE_JOB_MIN_REFRESH_SECONDS", "1") try: return max(0.0, float(raw_value or 0.0)) except (TypeError, ValueError): return 5.0 if reason == "heartbeat_autoresume" else 1.0 def _load_active_detect_job_snapshot(self, reason: str, now_ts: float): refresh_interval = self._active_detect_job_refresh_interval_seconds(reason) cached_job = getattr(self, "_cached_active_detect_job", None) last_refresh_at = float(getattr(self, "_last_active_detect_job_refresh_at", 0.0) or 0.0) if refresh_interval > 0 and cached_job is not None and now_ts - last_refresh_at < refresh_interval: return dict(cached_job or {}) active_job = self.db.get_active_detect_job() or {} self._cached_active_detect_job = dict(active_job or {}) self._last_active_detect_job_refresh_at = now_ts return dict(active_job or {}) def _autoresume_same_job_restart_grace_seconds(self) -> float: return max( 5.0, float(os.getenv("DOMAINCHECK_AUTORESUME_SAME_JOB_RESTART_GRACE_SECONDS", "30") or 30), ) def _bootstrap_autoresume_lock_ttl_seconds(self) -> int: return max( 5, int(os.getenv("DOMAINCHECK_BOOTSTRAP_AUTORESUME_LOCK_TTL_SECONDS", "30") or 30), ) def _acquire_bootstrap_autoresume_lock(self, *, job_id: int, job_code: str = "") -> bool: if not self.use_redis or self.redis_client is None: return True try: normalized_job_id = int(job_id or 0) except Exception: normalized_job_id = 0 if normalized_job_id <= 0: return True lock_action = f"bootstrap-autoresume:{normalized_job_id}:{str(job_code or '').strip() or '-'}" lock_key = self._shared_job_maintenance_lock_key(lock_action) if not lock_key: return True owner_token = self._job_maintenance_owner_token(lock_action) try: acquired = self.redis_client.set( lock_key, owner_token, nx=True, ex=self._bootstrap_autoresume_lock_ttl_seconds(), ) return bool(acquired) except Exception as exc: logger.debug(f"获取 bootstrap 自动回挂锁失败: action={lock_action}, error={exc}") return True def _should_skip_same_job_autoresume(self, *, active_job: dict, now_ts: float) -> tuple[bool, str]: incoming_job_code = str(active_job.get("job_code") or "").strip() incoming_job_id = active_job.get("id") try: normalized_incoming_job_id = int(incoming_job_id or 0) except Exception: normalized_incoming_job_id = 0 current_job_code = str(getattr(self, "current_job_code", "") or "").strip() current_job_id = getattr(self, "current_job_id", None) try: normalized_current_job_id = int(current_job_id or 0) except Exception: normalized_current_job_id = 0 same_job = bool( incoming_job_code and current_job_code and incoming_job_code == current_job_code ) or bool( normalized_incoming_job_id > 0 and normalized_current_job_id > 0 and normalized_incoming_job_id == normalized_current_job_id ) if not same_job: return False, "" live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) if live_active_threads > 0: return True, f"same_job_threads_active:{live_active_threads}" handoff_job_id = self._restart_release_handoff_job_id_active() if ( handoff_job_id not in (None, "", 0, "0") and normalized_incoming_job_id > 0 and int(handoff_job_id) == normalized_incoming_job_id ): return True, "same_job_restart_handoff_active" recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0) recent_start_grace_seconds = self._autoresume_same_job_restart_grace_seconds() recent_start_elapsed = max(0.0, now_ts - recent_start_at) if recent_start_at > 0 else 0.0 if recent_start_at > 0 and recent_start_elapsed < recent_start_grace_seconds: return True, f"same_job_recent_start:{int(recent_start_elapsed)}s" return False, "" def _should_skip_explicit_scope_autoresume(self, *, active_job: dict) -> tuple[bool, str]: explicit_job_id = self._explicit_claim_scope_job_id_active() if explicit_job_id in (None, "", 0, "0"): return False, "" try: normalized_explicit_job_id = int(explicit_job_id or 0) except Exception: return False, "" try: normalized_incoming_job_id = int(active_job.get("id") or 0) except Exception: normalized_incoming_job_id = 0 if normalized_explicit_job_id <= 0 or normalized_incoming_job_id <= 0: return False, "" if normalized_explicit_job_id != normalized_incoming_job_id: return False, "" explicit_job_code = str(getattr(self, "_explicit_claim_scope_job_code", "") or "").strip() explicit_source = str(getattr(self, "_explicit_claim_scope_source", "") or "").strip() or "unknown" detail_parts = [f"explicit_scope_active:{normalized_explicit_job_id}"] if explicit_job_code: detail_parts.append(explicit_job_code) detail_parts.append(explicit_source) return True, ":".join(detail_parts) def _ignored_targeted_job_window_seconds(self) -> float: return max( 30.0, float(os.getenv("DOMAINCHECK_IGNORED_TARGETED_JOB_WINDOW_SECONDS", "180") or 180), ) def _remember_ignored_targeted_control(self, control_payload=None) -> None: if not isinstance(control_payload, dict): return if control_targets_current_worker(control_payload): return target_node_codes = control_target_node_codes(control_payload) if not target_node_codes: return job_id = self._incoming_job_id(control_payload) job_code = self._incoming_job_code(control_payload) if job_id in (None, "", 0, "0") or not job_code: return self._ignored_targeted_job_id = int(job_id) self._ignored_targeted_job_code = str(job_code or "").strip() self._ignored_targeted_job_until = time.time() + self._ignored_targeted_job_window_seconds() self._ignored_targeted_job_targets = ",".join(target_node_codes) def _clear_ignored_targeted_job(self, *, reason: str = "") -> None: self._ignored_targeted_job_id = None self._ignored_targeted_job_code = "" self._ignored_targeted_job_until = 0.0 self._ignored_targeted_job_targets = "" def _should_skip_ignored_targeted_job_autoresume(self, *, active_job: dict) -> tuple[bool, str]: ignored_job_id = getattr(self, "_ignored_targeted_job_id", None) if ignored_job_id in (None, "", 0, "0"): return False, "" if time.time() >= float(getattr(self, "_ignored_targeted_job_until", 0.0) or 0.0): self._clear_ignored_targeted_job(reason="expired") return False, "" try: active_job_id = int(active_job.get("id") or 0) except Exception: active_job_id = 0 try: normalized_ignored_job_id = int(ignored_job_id or 0) except Exception: normalized_ignored_job_id = 0 if active_job_id <= 0 or normalized_ignored_job_id <= 0: return False, "" if active_job_id != normalized_ignored_job_id: return False, "" ignored_job_code = str(getattr(self, "_ignored_targeted_job_code", "") or "").strip() ignored_targets = str(getattr(self, "_ignored_targeted_job_targets", "") or "").strip() or "unknown" detail_parts = [f"ignored_targeted_job:{normalized_ignored_job_id}"] if ignored_job_code: detail_parts.append(ignored_job_code) detail_parts.append(ignored_targets) return True, ":".join(detail_parts) def _resume_active_detect_job_if_needed(self, reason: str = "worker_bootstrap"): now_ts = time.time() stale_detect = False autoresume_reasons = { "heartbeat_autoresume", "service_runtime_bootstrap", "redis_subscription_bootstrap", } node_role = str(getattr(config, "NODE_ROLE", "") or "").strip() is_primary_worker = self._is_primary_job_maintenance_worker() tail_handoff_probe_enabled = self._runtime_bool_override( "tail_handoff_autoresume_enabled", "DOMAINCHECK_TAIL_HANDOFF_AUTORESUME", default=True, ) def log_skip(detail: str): if now_ts - float(getattr(self, "_last_autoresume_skip_log_at", 0.0) or 0.0) >= 15.0: logger.info(f"自动回挂跳过: reason={reason}, detail={detail}") self._last_autoresume_skip_log_at = now_ts if ( node_role == "control" and not self._runtime_bool_override( "control_node_autoresume_enabled", "DOMAINCHECK_CONTROL_NODE_AUTORESUME", default=False, ) ): log_skip("control_node_autoresume_disabled") return False if ( reason in autoresume_reasons and node_role != "control" and not is_primary_worker and not tail_handoff_probe_enabled ): log_skip("non_primary_autoresume_worker") return False if not self.running: log_skip("worker_not_running") return False if self._consume_pending_restart_request(trigger_reason=reason): return True if self.stop_requested: log_skip("stop_requested") return False explicit_start_grace_seconds = max( 15.0, float(os.getenv("DOMAINCHECK_AUTORESUME_EXPLICIT_START_GRACE_SECONDS", "120") or 120), ) last_explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0) if ( last_explicit_start_at > 0 and now_ts - last_explicit_start_at < explicit_start_grace_seconds ): log_skip( "recent_explicit_start_signal" ) return False if self.detecting: stale_detect, stale_reason = self._is_stale_detect_session() if not stale_detect: log_skip(stale_reason or "already_detecting") return False logger.warning(f"检测运行态疑似空转，允许自动回挂接管: {stale_reason}") thread = self.detect_command_thread if thread and thread.is_alive() and not stale_detect: log_skip("detect_command_thread_alive") return False try: active_job = self._load_active_detect_job_snapshot(reason, now_ts) except Exception as e: logger.debug(f"读取当前检测任务失败，跳过自动回挂: {e}") return False job_id = active_job.get("id") job_code = str(active_job.get("job_code") or "").strip() task_mode = str(active_job.get("task_mode") or "").strip() if not job_id or not job_code: log_skip("active_job_missing_id_or_code") return False items_pending = int(active_job.get("items_pending", 0) or 0) items_claimed = int(active_job.get("items_claimed", 0) or 0) items_running = int(active_job.get("items_running", 0) or 0) selection_reason = str(active_job.get("selection_reason") or "").strip() if selection_reason == "running_job_stalled" or bool(active_job.get("running_job_stalled")): self._maybe_recycle_stalled_job_items( job_id, reason=f"autoresume:{reason or 'unspecified'}", ) log_skip( "running_job_stalled" ) return False tail_handoff_candidate = bool(active_job.get("tail_handoff_candidate")) or selection_reason == "tail_handoff_pending" if items_pending <= 0 and items_claimed <= 0 and items_running <= 0: log_skip("active_job_empty") return False if reason == "heartbeat_autoresume": should_skip_targeted_job, targeted_job_detail = self._should_skip_ignored_targeted_job_autoresume( active_job=active_job, ) if should_skip_targeted_job: log_skip(targeted_job_detail or "ignored_targeted_job") return False should_skip_explicit_scope, explicit_scope_detail = self._should_skip_explicit_scope_autoresume( active_job=active_job, ) if should_skip_explicit_scope: log_skip(explicit_scope_detail or "explicit_scope_autoresume_guard") return False should_skip_same_job, skip_detail = self._should_skip_same_job_autoresume( active_job=active_job, now_ts=now_ts, ) if should_skip_same_job: log_skip(skip_detail or "same_job_autoresume_guard") return False if reason in {"service_runtime_bootstrap", "redis_subscription_bootstrap"}: if not self._acquire_bootstrap_autoresume_lock(job_id=int(job_id), job_code=job_code): log_skip("bootstrap_autoresume_lock_held") return False if ( reason in autoresume_reasons and node_role != "control" and not is_primary_worker and not tail_handoff_candidate ): log_skip("non_primary_autoresume_worker") return False logger.warning( f"检测 Worker 空闲但发现活动任务，准备自动回挂: " f"job_code={job_code}, pending={items_pending}, claimed={items_claimed}, " f"running={items_running}, reason={reason}, selection={selection_reason or '-'}" ) started = self.start_detection_async( source="auto-resume", control_payload={ "source": reason, "job_id": int(job_id), "job_code": job_code, "task_mode": task_mode, "target_job_id": int(job_id), "target_job_code": job_code, "target_task_mode": task_mode, "selection_reason": selection_reason, "tail_handoff_candidate": tail_handoff_candidate, }, ) if started and now_ts - float(getattr(self, "_last_autoresume_success_log_at", 0.0) or 0.0) >= 5.0: logger.warning( f"自动回挂已触发: reason={reason}, job_code={job_code}, " f"pending={items_pending}, claimed={items_claimed}, running={items_running}, " f"selection={selection_reason or '-'}" ) self._last_autoresume_success_log_at = now_ts return started def activate_service_runtime(self): if not self.running: self.running = True logger.info( f"激活 Worker 服务态运行时: node={config.NODE_CODE}, " f"thread_count={self.thread_count}, proxy_enabled={1 if self.proxy_config.get('proxy_enable', False) else 0}" ) if self.use_redis: thread = getattr(self, "redis_sub_thread", None) if not thread or not thread.is_alive(): self.redis_sub_thread = threading.Thread( target=self.start_redis_subscription, name="RedisSubscriptionThread", daemon=True, ) self.redis_sub_thread.start() logger.info("Redis 配置/控制订阅线程已启动") try: self._consume_pending_control_command() except Exception as e: logger.warning(f"服务态启动时消费待执行控制指令失败: {e}") try: self._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") except Exception as e: logger.warning(f"服务态启动时自动回挂活动任务失败: {e}") def _update_runtime_state(self, phase: str, detail: str, **extra): """ 更新 Redis 中的检测运行态，供 Web 后台读取。 """ volatile_keys = { "active_threads", "current_load", "cycle_token", "job_id", "job_code", } previous_extra = dict(getattr(self, "_last_runtime_extra", {}) or {}) merged_extra = {key: value for key, value in previous_extra.items() if key not in volatile_keys} merged_extra.update({key: value for key, value in dict(extra or {}).items() if key not in volatile_keys}) if "max_threads" not in merged_extra: merged_extra["max_threads"] = int(getattr(self, "thread_count", 0) or 0) with self._domain_thread_counter_lock: live_active_threads = int(self._active_domain_threads or 0) explicit_active_threads = (extra or {}).get("active_threads") if not bool(self.detecting): effective_active_threads = 0 elif explicit_active_threads in (None, ""): effective_active_threads = live_active_threads else: effective_active_threads = max(int(explicit_active_threads or 0), live_active_threads) merged_extra["active_threads"] = effective_active_threads waiting_for_dispatch = bool(self.detecting and effective_active_threads <= 0) merged_extra["waiting_for_dispatch"] = waiting_for_dispatch explicit_current_load = (extra or {}).get("current_load") default_current_load = effective_active_threads if effective_active_threads > 0 else 0 if explicit_current_load in (None, ""): merged_extra["current_load"] = default_current_load else: merged_extra["current_load"] = max(int(explicit_current_load or 0), default_current_load) self._last_runtime_phase = phase self._last_runtime_detail = detail self._last_runtime_extra = merged_extra if not self.use_redis or self.redis_client is None: return with self.proxy_pool_lock: available_proxy_count = len(self.proxy_pool) payload_warning = "" if self.proxy_config.get('proxy_enable', False) and available_proxy_count <= 0: if ( self.proxy_last_refresh_status and self.proxy_last_refresh_status not in {"未刷新", "代理未启用", "未配置代理池链接"} and "刷新成功" not in self.proxy_last_refresh_status ): payload_warning = self.proxy_last_refresh_status payload = { "node_code": config.NODE_CODE, "phase": phase, "detail": detail, "service_running": bool(self.running), "detecting": bool(self.detecting), "stop_requested": bool(self.stop_requested), "available_proxy_count": available_proxy_count, "proxy_last_refresh_status": self.proxy_last_refresh_status, "proxy_last_refresh_time": self.proxy_last_refresh_time.strftime("%Y-%m-%d %H:%M:%S") if self.proxy_last_refresh_time else "", "proxy_last_refresh_source_count": int(self.proxy_last_refresh_source_count or 0), "proxy_last_refresh_total_items": int(self.proxy_last_refresh_total_items or 0), "proxy_last_validated_count": int(self.proxy_last_validated_count or 0), "proxy_last_available_count": int(self.proxy_last_available_count or 0), "proxy_last_source_stats": list(self.proxy_last_source_stats or []), "recent_warning": payload_warning, "updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } payload.update(merged_extra) if self.current_cycle_token and not payload.get("cycle_token"): payload["cycle_token"] = self.current_cycle_token if self.current_job_id and not payload.get("job_id"): payload["job_id"] = self.current_job_id if self.current_job_code and not payload.get("job_code"): payload["job_code"] = self.current_job_code now_ts = time.time() runtime_state_busy_phases = {"preparing", "refreshing_proxy", "fetching", "running", "completing"} runtime_state_min_interval = max( 0.05, float(os.getenv("DOMAINCHECK_RUNTIME_STATE_MIN_INTERVAL", "0.25") or 0.25), ) last_push_phase = str(getattr(self, "_last_runtime_state_push_phase", "") or "") last_push_at = float(getattr(self, "_last_runtime_state_push_at", 0.0) or 0.0) if ( phase in runtime_state_busy_phases and phase == last_push_phase and now_ts - last_push_at < runtime_state_min_interval ): return try: self.redis_client.set(runtime_state_key(), json.dumps(payload, ensure_ascii=False)) except Exception as e: logger.debug(f"写入检测运行态失败: {e}") try: actual_active_threads = int(payload.get("active_threads", 0) or 0) max_threads = int(payload.get("max_threads", getattr(self, "thread_count", 0)) or 0) current_load = actual_active_threads self.db.register_cluster_node( config.NODE_CODE, config.NODE_REGION, config.NODE_ROLE, status="busy" if payload.get("detecting") else "online", current_load=current_load, metadata={ "phase": phase, "detail": detail, "detecting": bool(payload.get("detecting")), "waiting_for_dispatch": bool(payload.get("waiting_for_dispatch")), "available_proxy_count": int(payload.get("available_proxy_count", 0) or 0), "active_threads": actual_active_threads, "max_threads": max_threads, "cycle_token": str(payload.get("cycle_token") or ""), "job_id": payload.get("job_id"), "job_code": str(payload.get("job_code") or ""), "proxy_runtime": { "sources": int(payload.get("proxy_last_refresh_source_count", 0) or 0), "raw_items": int(payload.get("proxy_last_refresh_total_items", 0) or 0), "validated": int(payload.get("proxy_last_validated_count", 0) or 0), "available": int(payload.get("proxy_last_available_count", 0) or 0), }, }, ) except Exception as e: logger.debug(f"更新检测节点心跳失败: {e}") self._last_runtime_state_push_at = now_ts self._last_runtime_state_push_phase = phase def _mark_detection_phase(self, phase: str, detail: str, **extra): if phase not in {"idle", "completed", "stopped"}: self._note_detection_activity() self._update_runtime_state(phase, detail, **extra) def _change_active_domain_threads(self, delta: int): with self._domain_thread_counter_lock: self._active_domain_threads = max(0, int(self._active_domain_threads or 0) + int(delta)) if self._active_domain_threads > 0: self._note_detection_activity() return self._active_domain_threads def _get_active_domain_threads(self): with self._domain_thread_counter_lock: return int(self._active_domain_threads or 0) def _note_detection_activity(self): now_ts = time.time() self.detect_command_last_activity_at = now_ts self._last_worker_activity_at = now_ts def _note_domain_started(self): now_ts = time.time() self._last_domain_started_at = now_ts self.detect_command_last_activity_at = now_ts self._last_worker_activity_at = now_ts def _note_domain_result(self): now_ts = time.time() self._last_domain_result_at = now_ts self.detect_command_last_activity_at = now_ts self._last_worker_activity_at = now_ts def _idle_worker_recycle_enabled(self): return self._env_flag_enabled("DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED", default=False) def _idle_worker_recycle_after_seconds(self): try: return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS", "180") or 180.0)) except Exception: return 180.0 def _idle_worker_recycle_min_uptime_seconds(self): try: return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS", "300") or 300.0)) except Exception: return 300.0 def _idle_worker_recycle_explicit_start_grace_seconds(self): try: return max(10.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS", "90") or 90.0)) except Exception: return 90.0 def _idle_worker_recycle_jitter_seconds(self): try: return max(0.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS", "45") or 45.0)) except Exception: return 45.0 def _idle_worker_recycle_jitter_offset(self): jitter_window = float(self._idle_worker_recycle_jitter_seconds() or 0.0) if jitter_window <= 0: return 0.0 node_code = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid()) spread = sum(ord(ch) for ch in node_code) % 1000 return (spread / 1000.0) * jitter_window def _is_recyclable_idle_worker_instance(self): if str(os.getenv("WORKER_MODE", "") or "").strip() != "linux-systemd": return False parent_node_code = self._worker_parent_node_code() node_code = str(getattr(config, "NODE_CODE", "") or "").strip() if not parent_node_code or not node_code: return False return node_code != parent_node_code and node_code.startswith(f"{parent_node_code}-") def _idle_worker_waiting_for_dispatch(self): runtime_extra = dict(getattr(self, "_last_runtime_extra", {}) or {}) if bool(runtime_extra.get("waiting_for_dispatch", False)): return True return bool(self.detecting and self._get_active_domain_threads() <= 0) def _idle_worker_seconds(self): last_activity_at = max( float(getattr(self, "_last_worker_activity_at", 0.0) or 0.0), float(getattr(self, "detect_command_last_activity_at", 0.0) or 0.0), float(getattr(self, "_last_domain_started_at", 0.0) or 0.0), float(getattr(self, "_last_domain_result_at", 0.0) or 0.0), ) if last_activity_at <= 0: last_activity_at = float(getattr(self, "_worker_started_at", 0.0) or time.time()) return max(0.0, time.time() - last_activity_at) def _trigger_idle_worker_recycle(self, *, trigger_reason: str, idle_seconds: float): if self._idle_recycle_requested: return False self._idle_recycle_requested = True detail = ( f"子 Worker 空闲过久，触发生命周期回收: idle={int(idle_seconds)}s, " f"trigger={trigger_reason or 'unknown'}" ) logger.warning(detail) try: self._sync_worker_log_event( "空闲 Worker 生命周期回收", level='warning', payload={ "idle_seconds": int(idle_seconds), "trigger_reason": str(trigger_reason or "").strip() or "unknown", "node_code": str(getattr(config, "NODE_CODE", "") or "").strip(), }, mode='key', ) except Exception: pass try: self._update_runtime_state( "restarting", detail, active_threads=0, current_load=0, recycle_reason=str(trigger_reason or "").strip() or "idle", ) except Exception: pass try: self._flush_pending_running_marks(force=True, batch_limit=2048) self._flush_pending_job_releases(force=True, batch_limit=2048) self._flush_pending_job_finalizations(force=True, batch_limit=2048) self._flush_pending_domain_status_updates(force=True, batch_limit=2048) self._flush_pending_domain_completions(force=True, batch_limit=2048) self._flush_pending_review_status_updates(force=True, batch_limit=2048) self._flush_completed_futures(force=True, batch_limit=2048) except Exception as exc: logger.debug(f"空闲 Worker 回收前 flush 失败: {exc}") self.stop_requested = True self.detecting = False self.running = False self._runtime_heartbeat_stop.set() self._worker_log_sync_stop.set() try: self._worker_log_sync_queue.put_nowait(None) except Exception: pass time.sleep(0.2) os.kill(os.getpid(), signal.SIGTERM) return True def _maybe_recycle_idle_worker(self, *, trigger_reason: str = ""): if self._idle_recycle_requested: return False if not self._idle_worker_recycle_enabled(): return False if not self._is_recyclable_idle_worker_instance(): return False if not self.running or self.stop_requested: return False if self._get_active_domain_threads() > 0: return False if not self._idle_worker_waiting_for_dispatch(): return False if time.time() - float(getattr(self, "_worker_started_at", 0.0) or 0.0) < self._idle_worker_recycle_min_uptime_seconds(): return False explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0) if explicit_start_at > 0 and time.time() - explicit_start_at < self._idle_worker_recycle_explicit_start_grace_seconds(): return False idle_seconds = self._idle_worker_seconds() recycle_after = self._idle_worker_recycle_after_seconds() + self._idle_worker_recycle_jitter_offset() if idle_seconds < recycle_after: return False return self._trigger_idle_worker_recycle( trigger_reason=trigger_reason or "idle_waiting_for_dispatch", idle_seconds=idle_seconds, ) def _schedule_pending_restart(self, source: str, control_payload=None, reason: str = ""): self._pending_restart_source = str(source or "deferred-restart").strip() or "deferred-restart" self._pending_restart_payload = dict(control_payload or {}) self._pending_restart_reason = str(reason or "").strip() def _should_force_takeover_stale_session(self, stale_reason: str = "") -> bool: thread = self.detect_command_thread if not thread or not thread.is_alive(): return False with self._domain_thread_counter_lock: live_active_threads = int(self._active_domain_threads or 0) if live_active_threads > 0: return False last_activity_at = max( float(self.detect_command_last_activity_at or 0.0), float(self.detect_command_started_at or 0.0), float(getattr(self, "_last_domain_started_at", 0.0) or 0.0), float(getattr(self, "_last_domain_result_at", 0.0) or 0.0), ) idle_seconds = max(0.0, time.time() - last_activity_at) if last_activity_at > 0 else 0.0 thread_age_seconds = max(0.0, time.time() - float(self.detect_command_started_at or 0.0)) if idle_seconds >= 10 or thread_age_seconds >= 20: return True return bool(stale_reason) def _consume_pending_restart_request(self, trigger_reason: str = ""): with self.detect_lock: if self.detect_command_thread and self.detect_command_thread.is_alive(): return False source = str(self._pending_restart_source or "").strip() payload = dict(self._pending_restart_payload or {}) pending_reason = str(self._pending_restart_reason or "").strip() if not source and not payload: return False self._pending_restart_source = "" self._pending_restart_payload = None self._pending_restart_reason = "" logger.warning( "检测任务执行延期重启: " f"trigger={trigger_reason or 'unknown'}, source={source or 'deferred-restart'}, " f"detail={pending_reason or 'none'}" ) return self.start_detection_async(source=source or "deferred-restart", control_payload=payload) def _detection_session_abort_reason(self, session_id: int = 0) -> str: if not self.running: return "worker_stopped" normalized_session_id = int(session_id or 0) if normalized_session_id <= 0: return "" current_owner = int(getattr(self, "_detect_session_owner", 0) or 0) if current_owner > 0 and current_owner != normalized_session_id: return f"session_replaced:{current_owner}" return "" def _release_job_item_for_session_abort(self, job_item_id, claim_token, *, session_id: int = 0, domain_name: str = "", detail: str = ""): abort_reason = self._detection_session_abort_reason(session_id) if not abort_reason: return False try: normalized_job_item_id = int(job_item_id or 0) except Exception: normalized_job_item_id = 0 normalized_claim_token = str(claim_token or "").strip() if normalized_job_item_id <= 0 or not normalized_claim_token: return False normalized_detail = str(detail or "").strip() release_reason = f"released after session abort: {abort_reason}" if normalized_detail: release_reason = f"{release_reason} | {normalized_detail[:240]}" release_reason = release_reason[:1000] released = self._enqueue_job_release( normalized_job_item_id, normalized_claim_token, reason=release_reason, ) if released: pending_release_count = 0 with self._job_release_lock: pending_release_count = len(self._pending_job_releases) if pending_release_count >= 64: self._flush_pending_job_releases(batch_limit=512) logger.warning( f"检测会话已切换，任务项已登记回队列: domain={domain_name or 'unknown'}, " f"job_item_id={normalized_job_item_id}, reason={abort_reason}, " f"detail={str(detail or '').strip() or 'none'}" ) return released def _assert_detection_session_active(self, session_id: int = 0, *, domain_name: str = "", stage: str = "") -> None: abort_reason = self._detection_session_abort_reason(session_id) if not abort_reason: return normalized_stage = str(stage or "unknown").strip() or "unknown" normalized_domain_name = str(domain_name or "").strip() detail = f"检测会话已失效: reason={abort_reason} stage={normalized_stage}" if normalized_domain_name: detail = f"{detail} domain={normalized_domain_name}" raise RuntimeError(detail) def _incoming_job_code(self, control_payload=None) -> str: payload = control_payload or {} job_code = str(payload.get("job_code") or "").strip() if not job_code: job_code = str(payload.get("target_job_code") or "").strip() return job_code def _incoming_cycle_token(self, control_payload=None) -> str: payload = control_payload or {} return str(payload.get("cycle_token") or "").strip() def _incoming_control_source(self, control_payload=None) -> str: payload = control_payload or {} return str(payload.get("source") or "").strip() def _incoming_job_id(self, control_payload=None): payload = control_payload or {} # For domain_pipeline sync-pull commands, runtime/log identity may follow # the upstream projection job_code/job_id, but local queue ownership must # stay pinned to the ingested target_job_id. Claim/release/session scope # logic therefore prefers target_job_id when it exists. job_id = payload.get("target_job_id") if job_id in (None, "", 0, "0"): job_id = payload.get("job_id") try: return int(job_id) if job_id not in (None, "", 0, "0") else None except Exception: return None def _is_overlap_handoff_command(self, control_payload=None, *, source: str = "") -> bool: payload = control_payload or {} normalized_source = str(source or self._incoming_control_source(payload) or "").strip() if normalized_source == "overlap-handoff": return True return bool(payload.get("tail_handoff_candidate")) def _matches_active_detect_session(self, control_payload=None) -> bool: payload = control_payload or {} incoming_job_code = self._incoming_job_code(payload) incoming_job_id = self._incoming_job_id(payload) incoming_cycle_token = self._incoming_cycle_token(payload) current_job_code = str(self.current_job_code or "").strip() current_cycle_token = str(self.current_cycle_token or "").strip() current_job_id = getattr(self, "current_job_id", None) matches_job_code = bool( incoming_job_code and current_job_code and incoming_job_code == current_job_code ) matches_job_id = bool( incoming_job_id not in (None, 0) and current_job_id not in (None, 0) and int(incoming_job_id) == int(current_job_id) ) if not (matches_job_code or matches_job_id): return False if incoming_cycle_token and current_cycle_token and incoming_cycle_token != current_cycle_token: return False return True def _is_stale_detect_session(self, control_payload=None): thread = self.detect_command_thread if not thread or not thread.is_alive(): return False, "" live_active_threads = 0 with self._domain_thread_counter_lock: live_active_threads = int(self._active_domain_threads or 0) last_activity_at = max( float(self.detect_command_last_activity_at or 0.0), float(self.detect_command_started_at or 0.0), float(getattr(self, "_last_domain_started_at", 0.0) or 0.0), float(getattr(self, "_last_domain_result_at", 0.0) or 0.0), ) if last_activity_at <= 0: return False, "" idle_seconds = max(0.0, time.time() - last_activity_at) thread_age_seconds = max(0.0, time.time() - float(self.detect_command_started_at or 0.0)) last_result_at = float(getattr(self, "_last_domain_result_at", 0.0) or 0.0) result_idle_seconds = max(0.0, time.time() - last_result_at) if last_result_at > 0 else idle_seconds incoming_job_code = self._incoming_job_code(control_payload) incoming_cycle_token = self._incoming_cycle_token(control_payload) incoming_task_mode = str( (control_payload or {}).get("task_mode") or (control_payload or {}).get("target_task_mode") or "" ).strip() current_job_code = str(self.current_job_code or "").strip() current_cycle_token = str(self.current_cycle_token or "").strip() if self._matches_active_detect_session(control_payload): if idle_seconds < 180 and result_idle_seconds < 180: return False, ( "同任务同周期重复唤起已忽略: " f"job={incoming_job_code or current_job_code or 'unknown'} " f"cycle={incoming_cycle_token or current_cycle_token or 'none'}" ) session_scoped_to_job = bool( self._is_single_step_session_active() or incoming_task_mode == "single_step" ) job_switched = bool( session_scoped_to_job and incoming_job_code and current_job_code and incoming_job_code != current_job_code ) incoming_job_changed = bool( session_scoped_to_job and incoming_job_code and incoming_job_code != current_job_code ) low_activity_recoverable = live_active_threads <= 1 multi_thread_force_recover = incoming_job_changed and thread_age_seconds >= 30 and result_idle_seconds >= 30 if not low_activity_recoverable and not multi_thread_force_recover: return False, "" hold_for_proxy_shortage, hold_reason = self._should_hold_stale_detect_for_proxy_shortage( idle_seconds=idle_seconds, low_activity_recoverable=low_activity_recoverable, incoming_job_changed=job_switched or incoming_job_changed, ) if hold_for_proxy_shortage: return False, hold_reason if job_switched and idle_seconds >= 15: return True, ( f"检测指令线程仍停留在旧任务 {current_job_code}，" f"但新任务已切换为 {incoming_job_code}，且已空转 {int(idle_seconds)} 秒" ) if incoming_job_changed and idle_seconds >= 20: return True, ( f"检测指令线程未挂载新任务 {incoming_job_code}，" f"当前任务上下文为 {current_job_code or 'empty'}，且已空转 {int(idle_seconds)} 秒" ) if multi_thread_force_recover: return True, ( f"检测指令线程已运行 {int(thread_age_seconds)} 秒，最近 {int(result_idle_seconds)} 秒无域名产出，" f"且新任务 {incoming_job_code} 已到达，当前活跃线程 {live_active_threads}" ) if idle_seconds >= 45: return True, f"检测指令线程已空转 {int(idle_seconds)} 秒，当前活跃线程 {live_active_threads}" if thread_age_seconds >= 30 and idle_seconds >= 20: return True, ( f"检测指令线程已启动 {int(thread_age_seconds)} 秒，但最近 {int(idle_seconds)} 秒没有任何有效检测活动，当前活跃线程 {live_active_threads}" ) return False, "" def _set_active_cycle_context(self, control_payload=None): payload = control_payload or {} self._last_synced_worker_log = "" self.current_cycle_token = str(payload.get("cycle_token") or "").strip() job_id = payload.get("job_id") if job_id in (None, "", 0, "0"): job_id = payload.get("target_job_id") try: self.current_job_id = int(job_id) if job_id not in (None, "", 0, "0") else None except Exception: self.current_job_id = None job_code = str(payload.get("job_code") or "").strip() if not job_code: job_code = str(payload.get("target_job_code") or "").strip() self.current_job_code = job_code task_mode = str(payload.get("task_mode") or payload.get("target_task_mode") or "").strip() if not task_mode and str(job_code).startswith("step-"): task_mode = "single_step" self.current_job_task_mode = task_mode def _clear_active_cycle_context(self): self._last_synced_worker_log = "" self.current_cycle_token = "" self.current_job_id = None self.current_job_code = "" self.current_job_task_mode = "" self._clear_restart_release_handoff(reason="clear_active_cycle") self.detect_command_started_at = 0.0 self.detect_command_last_activity_at = 0.0 self._last_domain_started_at = 0.0 self._last_domain_result_at = 0.0 def _is_single_step_session_active(self): task_mode = str(getattr(self, "current_job_task_mode", "") or "").strip() if task_mode == "single_step": return True return str(getattr(self, "current_job_code", "") or "").strip().startswith("step-") def _shared_job_maintenance_scope(self): parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() scope_node = parent_node_code or str(getattr(config, "NODE_CODE", "") or "").strip() or "unknown" db_host = str(getattr(config, "DB_HOST", "") or "").strip() or "db" db_port = str(getattr(config, "DB_PORT", "") or "").strip() or "5432" db_name = str(getattr(config, "DB_DATABASE", "") or "").strip() or "domain" return f"{db_host}:{db_port}:{db_name}:{scope_node}" def _shared_job_maintenance_lock_key(self, action: str) -> str: if not self.use_redis or self.redis_client is None: return "" normalized_action = str(action or "").strip() if not normalized_action: return "" return f"domaincheck:job-maintenance:{normalized_action}:{self._shared_job_maintenance_scope()}" def _job_maintenance_owner_token(self, action: str) -> str: return ( f"{str(getattr(config, 'NODE_CODE', '') or '').strip()}:" f"{os.getpid()}:{threading.get_ident()}:{time.time_ns()}:{str(action or '').strip()}" ) def _acquire_shared_job_maintenance_lock(self, action: str, *, ttl_seconds: int = 15) -> str: redis_key = self._shared_job_maintenance_lock_key(action) if not redis_key: return "__local__" if self._is_primary_job_maintenance_worker() else "" owner_token = self._job_maintenance_owner_token(action) try: acquired = self.redis_client.set( redis_key, owner_token, nx=True, ex=max(1, int(ttl_seconds or 15)), ) except Exception as exc: logger.debug(f"获取共享任务维护锁失败: action={action}, error={exc}") return "" return owner_token if acquired else "" def _release_shared_job_maintenance_lock(self, action: str, owner_token: str) -> None: if not owner_token or owner_token == "__local__": return redis_key = self._shared_job_maintenance_lock_key(action) if not redis_key: return try: current_owner = self.redis_client.get(redis_key) if str(current_owner or "") == str(owner_token): self.redis_client.delete(redis_key) except Exception as exc: logger.debug(f"释放共享任务维护锁失败: action={action}, error={exc}") def _stalled_job_recycle_interval_seconds(self) -> float: return max( 15.0, float(os.getenv("DOMAINCHECK_STALLED_JOB_RECYCLE_INTERVAL_SECONDS", "60") or 60), ) def _stalled_job_recycle_stall_seconds(self) -> int: return max( 300, int(os.getenv("DOMAINCHECK_RUNNING_JOB_STALL_SECONDS", "900") or 900), ) def _maybe_recycle_stalled_job_items(self, job_id, *, reason: str = "") -> int: try: normalized_job_id = int(job_id or 0) except Exception: normalized_job_id = 0 if normalized_job_id <= 0: return 0 now_ts = time.time() last_attempts = getattr(self, "_last_stalled_job_recycle_attempt_at", None) if not isinstance(last_attempts, dict): last_attempts = {} self._last_stalled_job_recycle_attempt_at = last_attempts last_attempt_at = float(last_attempts.get(normalized_job_id, 0.0) or 0.0) if now_ts - last_attempt_at < self._stalled_job_recycle_interval_seconds(): return 0 last_attempts[normalized_job_id] = now_ts lock_action = f"recycle-stalled-job-items:{normalized_job_id}" lock_owner = self._acquire_shared_job_maintenance_lock(lock_action, ttl_seconds=15) if not lock_owner: return 0 try: recycled = int( self.db.recycle_stalled_detect_job_items( normalized_job_id, stall_seconds=self._stalled_job_recycle_stall_seconds(), ) or 0 ) if recycled: logger.warning( f"已定向回收长挂任务项: job_id={normalized_job_id}, " f"recycled={recycled}, reason={reason or 'unspecified'}" ) return recycled finally: self._release_shared_job_maintenance_lock(lock_action, lock_owner) def _is_primary_job_maintenance_worker(self) -> bool: node_code = str(getattr(config, "NODE_CODE", "") or "").strip() parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() if parent_node_code and node_code == parent_node_code: return True if parent_node_code and node_code == f"{parent_node_code}-a": return True return bool(node_code and node_code.endswith("-a")) def _maybe_release_node_job_items(self, *, reason: str = "", min_interval_seconds: float = 300.0) -> int: now_ts = time.time() last_release_at = float(getattr(self, "_last_job_item_release_at", 0.0) or 0.0) if min_interval_seconds > 0 and now_ts - last_release_at < float(min_interval_seconds): return 0 released = int(self.db.release_detect_job_items_for_node(config.NODE_CODE) or 0) self._last_job_item_release_at = now_ts if released: self._schedule_restart_release_handoff( released_count=released, reason=reason or "node_item_release", ) logger.warning( f"释放当前节点遗留任务项: count={released}, node={config.NODE_CODE}, " f"reason={reason or 'unspecified'}" ) return released def _restart_release_handoff_window_seconds(self) -> float: return max( 30.0, float(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_WINDOW_SECONDS", "240") or 240), ) def _restart_release_handoff_batch_limit(self, requested_limit: int) -> int: configured = int(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_BATCH_LIMIT", "128") or 128) safe_requested_limit = max(1, int(requested_limit or 1)) return max(1, min(safe_requested_limit, max(1, configured))) def _schedule_restart_release_handoff(self, *, released_count: int = 0, reason: str = "") -> None: if int(released_count or 0) <= 0: return job_id = getattr(self, "current_job_id", None) if job_id in (None, "", 0, "0"): return try: normalized_job_id = int(job_id) except Exception: return self._restart_release_handoff_job_id = normalized_job_id self._restart_release_handoff_job_code = str(getattr(self, "current_job_code", "") or "").strip() self._restart_release_handoff_until = time.time() + self._restart_release_handoff_window_seconds() self._restart_release_handoff_reason = str(reason or "").strip() logger.warning( f"登记重启尾批优先回收窗口: job_id={normalized_job_id}, " f"job_code={self._restart_release_handoff_job_code or '-'}, " f"released={int(released_count or 0)}, reason={self._restart_release_handoff_reason or 'unspecified'}" ) def _clear_restart_release_handoff(self, *, reason: str = "") -> None: job_id = getattr(self, "_restart_release_handoff_job_id", None) if job_id not in (None, "", 0, "0"): logger.info( f"清理重启尾批优先回收窗口: job_id={job_id}, " f"job_code={str(getattr(self, '_restart_release_handoff_job_code', '') or '').strip() or '-'}, " f"reason={str(reason or 'unspecified').strip() or 'unspecified'}" ) self._restart_release_handoff_job_id = None self._restart_release_handoff_job_code = "" self._restart_release_handoff_until = 0.0 self._restart_release_handoff_reason = "" def _restart_release_handoff_job_id_active(self): handoff_job_id = getattr(self, "_restart_release_handoff_job_id", None) if handoff_job_id in (None, "", 0, "0"): return None if time.time() >= float(getattr(self, "_restart_release_handoff_until", 0.0) or 0.0): self._clear_restart_release_handoff(reason="expired") return None current_job_id = getattr(self, "current_job_id", None) if current_job_id not in (None, "", 0, "0"): try: if int(current_job_id) != int(handoff_job_id): self._clear_restart_release_handoff(reason="job_changed") return None except Exception: self._clear_restart_release_handoff(reason="job_changed") return None try: return int(handoff_job_id) except Exception: self._clear_restart_release_handoff(reason="invalid_job_id") return None def _explicit_claim_scope_window_seconds(self) -> float: return max( 30.0, float(os.getenv("DOMAINCHECK_EXPLICIT_CLAIM_SCOPE_WINDOW_SECONDS", "180") or 180), ) def _schedule_explicit_claim_scope(self, control_payload=None, *, source: str = "") -> None: payload = dict(control_payload or {}) task_mode = str( payload.get("task_mode") or payload.get("target_task_mode") or "" ).strip() if task_mode == "single_step": return job_id = self._incoming_job_id(payload) if job_id in (None, "", 0, "0"): return try: normalized_job_id = int(job_id) except Exception: return job_code = self._incoming_job_code(payload) self._explicit_claim_scope_job_id = normalized_job_id self._explicit_claim_scope_job_code = str(job_code or "").strip() self._explicit_claim_scope_until = time.time() + self._explicit_claim_scope_window_seconds() self._explicit_claim_scope_source = str(source or "").strip() logger.warning( f"登记显式启动定向领取窗口: job_id={normalized_job_id}, " f"job_code={self._explicit_claim_scope_job_code or '-'}, " f"source={self._explicit_claim_scope_source or 'unknown'}" ) def _clear_explicit_claim_scope(self, *, reason: str = "") -> None: job_id = getattr(self, "_explicit_claim_scope_job_id", None) if job_id not in (None, "", 0, "0"): logger.info( f"清理显式启动定向领取窗口: job_id={job_id}, " f"job_code={str(getattr(self, '_explicit_claim_scope_job_code', '') or '').strip() or '-'}, " f"reason={str(reason or 'unspecified').strip() or 'unspecified'}" ) self._explicit_claim_scope_job_id = None self._explicit_claim_scope_job_code = "" self._explicit_claim_scope_until = 0.0 self._explicit_claim_scope_source = "" def _explicit_claim_scope_job_id_active(self): explicit_job_id = getattr(self, "_explicit_claim_scope_job_id", None) if explicit_job_id in (None, "", 0, "0"): return None if time.time() >= float(getattr(self, "_explicit_claim_scope_until", 0.0) or 0.0): self._clear_explicit_claim_scope(reason="expired") return None try: return int(explicit_job_id) except Exception: self._clear_explicit_claim_scope(reason="invalid_job_id") return None def _should_scope_claims_to_current_job(self): """ single_step 会话必须严格绑定当前 job； domain_pipeline 会话则应持续从全局队列补位，避免旧 job 占住 worker、 新 job 长时间堆在 pending。 """ if self._is_single_step_session_active(): return True if self._restart_release_handoff_job_id_active() not in (None, "", 0, "0"): return True return self._explicit_claim_scope_job_id_active() not in (None, "", 0, "0") def _current_scoped_claim_job_id(self): if self._is_single_step_session_active(): try: return int(self.current_job_id) if self.current_job_id not in (None, "", 0, "0") else None except Exception: return None handoff_job_id = self._restart_release_handoff_job_id_active() if handoff_job_id not in (None, "", 0, "0"): return handoff_job_id explicit_job_id = self._explicit_claim_scope_job_id_active() if explicit_job_id not in (None, "", 0, "0"): return explicit_job_id return None def _log_explicit_scope_claim_result(self, *, scoped_job_id, limit, rows_count, reason=""): try: normalized_job_id = int(scoped_job_id or 0) except Exception: normalized_job_id = 0 if normalized_job_id <= 0: return normalized_rows = max(0, int(rows_count or 0)) normalized_limit = max(0, int(limit or 0)) detail_reason = str(reason or "").strip() or ("claimed" if normalized_rows > 0 else "empty") log_key = f"{normalized_job_id}:{normalized_rows}:{detail_reason}" now_ts = time.time() min_interval_seconds = 1.0 if normalized_rows > 0 else 10.0 if ( log_key == str(getattr(self, "_last_explicit_scope_claim_log_key", "") or "") and now_ts - float(getattr(self, "_last_explicit_scope_claim_log_at", 0.0) or 0.0) < min_interval_seconds ): return self._last_explicit_scope_claim_log_at = now_ts self._last_explicit_scope_claim_log_key = log_key log_message = ( f"显式定向领取{'命中' if normalized_rows > 0 else '未命中'}: " f"scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} " f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} " f"| limit={normalized_limit} | rows={normalized_rows} | reason={detail_reason}" ) if normalized_rows > 0: logger.warning(log_message) else: logger.info(log_message) def _log_explicit_scope_probe(self, *, scoped_job_id, max_threads, refill_slots, claim_batch_size, pending_buffer): try: normalized_job_id = int(scoped_job_id or 0) except Exception: normalized_job_id = 0 if normalized_job_id <= 0: return now_ts = time.time() if now_ts - float(getattr(self, "_last_explicit_scope_probe_log_at", 0.0) or 0.0) < 5.0: return self._last_explicit_scope_probe_log_at = now_ts logger.warning( f"显式定向领取准备开始: scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} " f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} " f"| max_threads={max(0, int(max_threads or 0))} | refill_slots={max(0, int(refill_slots or 0))} " f"| claim_batch_size={max(0, int(claim_batch_size or 0))} | pending_buffer={max(0, int(pending_buffer or 0))}" ) def _log_job_status_refresh_probe(self, job_id): try: normalized_job_id = int(job_id or 0) except Exception: normalized_job_id = 0 if normalized_job_id <= 0: return now_ts = time.time() if now_ts - float(getattr(self, "_last_job_status_refresh_probe_log_at", 0.0) or 0.0) < 10.0: return self._last_job_status_refresh_probe_log_at = now_ts logger.info( f"准备刷新检测任务聚合状态: job_id={normalized_job_id} | current_job_code={self.current_job_code or '-'} " f"| detecting={1 if self.detecting else 0} | active_threads={self._get_active_domain_threads()}" ) def _recent_duplicate_start_window_seconds(self) -> float: return max( 15.0, float(os.getenv("DOMAINCHECK_RECENT_START_DUPLICATE_WINDOW_SECONDS", "120") or 120), ) def _record_detection_start_request(self, *, source: str = "", control_payload=None) -> None: now_ts = time.time() payload = dict(control_payload or {}) job_id = self._incoming_job_id(payload) job_code = self._incoming_job_code(payload) task_mode = str( payload.get("task_mode") or payload.get("target_task_mode") or "" ).strip() self._last_detection_start_at = now_ts self._last_detection_start_source = str(source or "").strip() self._last_detection_start_job_id = int(job_id) if job_id not in (None, 0) else None self._last_detection_start_job_code = job_code self._last_detection_start_task_mode = task_mode def _mark_explicit_start_signal(self, control_payload=None, *, source: str = "") -> None: self._last_explicit_start_signal_at = time.time() self._last_explicit_start_payload = { **dict(control_payload or {}), "_source": str(source or "").strip(), } self._schedule_explicit_claim_scope(control_payload, source=source) def _should_coalesce_recent_start(self, control_payload=None) -> tuple[bool, str]: now_ts = time.time() recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0) if recent_start_at <= 0: return False, "" if now_ts - recent_start_at > self._recent_duplicate_start_window_seconds(): return False, "" if self.stop_requested: return False, "" thread = self.detect_command_thread thread_alive = bool(thread and thread.is_alive()) if not bool(self.detecting) and not thread_alive: # A recent start alone is not enough to suppress a legitimate retry. # Once the previous session has fully stopped, repeated start_detection # for the same job must be allowed to re-open tail work. return False, "" stale_detect, _ = self._is_stale_detect_session(control_payload) if stale_detect: # A stale session must never suppress a restart for the same job. # Otherwise auto-resume loops forever on "already running" while # active_threads stays at zero. return False, "" payload = dict(control_payload or {}) incoming_job_code = self._incoming_job_code(payload) incoming_job_id = self._incoming_job_id(payload) incoming_task_mode = str( payload.get("task_mode") or payload.get("target_task_mode") or "" ).strip() recent_job_code = str(getattr(self, "_last_detection_start_job_code", "") or "").strip() recent_job_id = getattr(self, "_last_detection_start_job_id", None) recent_task_mode = str(getattr(self, "_last_detection_start_task_mode", "") or "").strip() if incoming_job_code and recent_job_code and incoming_job_code != recent_job_code: return False, "" if ( incoming_job_id not in (None, 0) and recent_job_id not in (None, 0) and int(incoming_job_id) != int(recent_job_id) ): return False, "" if incoming_task_mode == "single_step" and recent_task_mode and incoming_task_mode != recent_task_mode: return False, "" effective_job_code = incoming_job_code or recent_job_code or str(self.current_job_code or "").strip() effective_job_id = incoming_job_id or recent_job_id or self.current_job_id if not effective_job_code and effective_job_id in (None, 0): return False, "" return True, ( "最近已启动同一轮检测，会话仍在稳定中，忽略重复启动: " f"job={effective_job_code or effective_job_id or 'none'} " f"source={str(getattr(self, '_last_detection_start_source', '') or '').strip() or 'unknown'}" ) def start_detection_async(self, source: str = "remote", control_payload=None): """ 异步启动一次检测任务，避免阻塞 Redis 订阅线程。 """ with self.detect_lock: recent_duplicate_start, recent_duplicate_message = self._should_coalesce_recent_start(control_payload) if recent_duplicate_start: logger.info(recent_duplicate_message) self._update_runtime_state( "running", recent_duplicate_message, source=source, cycle_token=self.current_cycle_token or self._incoming_cycle_token(control_payload), job_id=self.current_job_id or self._incoming_job_id(control_payload), job_code=str(self.current_job_code or "").strip() or self._incoming_job_code(control_payload), ) return True if self.detect_command_thread and self.detect_command_thread.is_alive(): duplicate_session_command = self._matches_active_detect_session(control_payload) stale_detect, stale_reason = self._is_stale_detect_session(control_payload) if not stale_detect: incoming_job_code = self._incoming_job_code(control_payload) incoming_cycle_token = self._incoming_cycle_token(control_payload) incoming_job_id = self._incoming_job_id(control_payload) overlap_handoff_requested = self._is_overlap_handoff_command(control_payload, source=source) current_job_code = str(self.current_job_code or "").strip() incoming_task_mode = str( (control_payload or {}).get("task_mode") or (control_payload or {}).get("target_task_mode") or "" ).strip() if duplicate_session_command: message = ( "收到同任务同周期重复启动指令，保持当前检测会话: " f"job={current_job_code or incoming_job_code or 'none'} " f"cycle={self.current_cycle_token or incoming_cycle_token or 'none'}" ) logger.info(message) self._update_runtime_state( "running", message, source=source, cycle_token=self.current_cycle_token or incoming_cycle_token, job_id=self.current_job_id or incoming_job_id, job_code=current_job_code or incoming_job_code, ) return True if ( incoming_task_mode != "single_step" and not self._is_single_step_session_active() and incoming_job_code and incoming_job_code != current_job_code ): explicit_scope_job_id = self._explicit_claim_scope_job_id_active() explicit_switch_requested = bool( explicit_scope_job_id not in (None, "", 0, "0") and incoming_job_id not in (None, "", 0, "0") and int(explicit_scope_job_id) == int(incoming_job_id) ) live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) if overlap_handoff_requested and live_active_threads > 0: message = ( "收到 overlap handoff 启动指令，但当前会话仍有活跃线程，忽略跨 job 抢占: " f"current={current_job_code or 'none'} -> incoming={incoming_job_code} " f"active_threads={live_active_threads}" ) logger.info(message) if explicit_switch_requested: self._clear_explicit_claim_scope(reason="overlap_busy_ignore") self._update_runtime_state( "running", message, source=source, cycle_token=self.current_cycle_token or incoming_cycle_token, job_id=self.current_job_id or incoming_job_id, job_code=current_job_code or incoming_job_code, ) return True if explicit_switch_requested: message = ( "检测任务已在运行，收到显式 pipeline 切换指令，" f"登记安全重启接棒: current={current_job_code or 'none'} -> incoming={incoming_job_code}" ) logger.warning(message) self.stop_requested = True self._schedule_pending_restart(source, control_payload, message) self._update_runtime_state( "restarting", message, source=source, cycle_token=incoming_cycle_token or self.current_cycle_token, job_id=incoming_job_id, job_code=incoming_job_code, ) return True message = ( "检测任务已在运行，已接收新的 pipeline 唤醒指令，" f"继续由当前 worker pool 补位处理: current={current_job_code or 'none'} -> incoming={incoming_job_code}" ) logger.info(message) self._update_runtime_state("running", message) return True logger.warning("收到启动检测指令，但检测任务已在运行，忽略重复启动") self._update_runtime_state("running", "检测任务已在运行，忽略重复启动") return False logger.warning(f"检测任务线程疑似卡死，准备回收后重新启动: {stale_reason}") self.stop_requested = True recovery_deadline = time.time() + 8 while self.detect_command_thread and self.detect_command_thread.is_alive() and time.time() < recovery_deadline: time.sleep(0.2) if self.detect_command_thread and self.detect_command_thread.is_alive(): if self._should_force_takeover_stale_session(stale_reason): logger.warning(f"检测任务线程超时未退出，执行强制接管: {stale_reason}") self.detect_command_thread = None self.detecting = False self._clear_active_cycle_context() self._update_runtime_state("restarting", f"检测线程疑似卡死，执行强制接管: {stale_reason}") else: self._schedule_pending_restart(source, control_payload, stale_reason) self._update_runtime_state("restarting", f"检测线程疑似卡死，已登记延期重启: {stale_reason}") return True self.stop_requested = False self.detect_command_started_at = time.time() self.detect_command_last_activity_at = self.detect_command_started_at self._detect_session_seq = int(self._detect_session_seq or 0) + 1 session_id = int(self._detect_session_seq or 0) self._detect_session_owner = session_id self._record_detection_start_request(source=source, control_payload=control_payload) self.detect_command_thread = threading.Thread( target=self._run_detection_session, kwargs={ "source": source, "control_payload": dict(control_payload or {}), "session_id": session_id, }, name="RemoteDetectCommandThread", daemon=True, ) self.detect_command_thread.start() logger.info(f"已接受检测启动指令，来源: {source}") self._set_active_cycle_context(control_payload) self._update_runtime_state( "starting", f"已接受检测启动指令，来源: {source}", source=source, cycle_token=self.current_cycle_token, job_id=self.current_job_id, job_code=self.current_job_code, ) return True def _run_detection_session(self, source: str = "remote", control_payload=None, session_id: int = 0): self._set_active_cycle_context(control_payload) with self.detect_lock: current_owner = int(getattr(self, "_detect_session_owner", 0) or 0) if session_id and current_owner and session_id != current_owner: logger.warning( f"检测任务会话启动时发现 owner 已切换，忽略旧会话: session_id={session_id}, current_owner={current_owner}" ) return self.detecting = True try: logger.info(f"开始执行远程检测任务，来源: {source}") self._sync_worker_log_event(f"开始执行检测任务，来源: {source}", payload={"source": source}, mode='key') self._update_runtime_state( "starting", f"开始执行检测任务，来源: {source}", source=source, cycle_token=self.current_cycle_token, job_id=self.current_job_id, job_code=self.current_job_code, ) self.start_detection(session_id=session_id) except Exception as e: logger.error(f"远程检测任务执行失败: {e}") import traceback logger.error(traceback.format_exc()) self._update_runtime_state( "failed", f"检测任务执行失败: {e}", source=source, cycle_token=self.current_cycle_token, job_id=self.current_job_id, job_code=self.current_job_code, ) finally: current_thread = threading.current_thread() with self.detect_lock: current_owner = int(getattr(self, "_detect_session_owner", 0) or 0) is_owner = not session_id or current_owner in {0, session_id} if is_owner: self.detecting = False if session_id: self._detect_session_owner = 0 if self.detect_command_thread is current_thread or not ( self.detect_command_thread and self.detect_command_thread.is_alive() ): self.detect_command_thread = None if not is_owner: logger.warning( f"检测任务旧会话结束，忽略清理: session_id={session_id}, current_owner={current_owner}" ) return restarted_from_pending = False if self.running: try: restarted_from_pending = bool( self._consume_pending_restart_request(trigger_reason="session_finalize") ) except Exception as restart_error: logger.warning(f"检测任务延期重启触发失败: {restart_error}") if self.running and not self.stop_requested and not restarted_from_pending: self._clear_active_cycle_context() self._update_runtime_state( "idle", "检测任务结束，Worker 保持待命", source=source, ) try: self._maybe_recycle_idle_worker(trigger_reason="session_finalize") except Exception as recycle_error: logger.debug(f"检测任务结束后空闲回收检查失败: {recycle_error}") else: self._clear_active_cycle_context() def request_stop_detection(self, source: str = "remote"): """ 请求停止当前检测任务，但保留 Worker 进程存活。 """ if not self.detecting and not (self.detect_command_thread and self.detect_command_thread.is_alive()): logger.info("收到停止检测指令，但当前没有运行中的检测任务") self._update_runtime_state("idle", "当前没有运行中的检测任务") return False logger.info(f"收到停止检测指令，来源: {source}") self._sync_worker_log_event(f"收到停止检测指令，来源: {source}", payload={"source": source}, mode='key') self.stop_requested = True self._update_runtime_state( "stopping", f"收到停止检测指令，等待当前批次安全退出，来源: {source}", cycle_token=self.current_cycle_token, job_id=self.current_job_id, job_code=self.current_job_code, ) return True def _acknowledge_pending_control_command(self, control_payload): if not self.use_redis or self.redis_client is None: return request_id = str((control_payload or {}).get("request_id", "")).strip() if not request_id: return pending_keys = [pending_control_key()] if pending_keys[0] != PENDING_CONTROL_KEY: pending_keys.append(PENDING_CONTROL_KEY) for key in pending_keys: try: raw_pending = self.redis_client.get(key) if not raw_pending: continue pending_payload = json.loads(raw_pending) pending_request_id = str((pending_payload or {}).get("request_id", "")).strip() if pending_request_id and pending_request_id == request_id: self.redis_client.delete(key) except Exception as e: logger.debug(f"确认待执行控制指令失败: key={key}, error={e}") def _handle_control_message(self, payload): try: control_payload = json.loads(payload) if isinstance(payload, str) else payload except Exception: control_payload = {"action": str(payload)} if not control_targets_current_worker(control_payload): self._remember_ignored_targeted_control(control_payload) target_node_codes = ",".join(control_target_node_codes(control_payload)) or "unknown" logger.info( f"忽略发往其他 Worker 实例的控制指令: current={config.NODE_CODE}, targets={target_node_codes}" ) return self._acknowledge_pending_control_command(control_payload) action = str(control_payload.get("action", "")).strip() if action == "start_detection": self._mark_explicit_start_signal(control_payload, source="redis-control") self.start_detection_async(source="redis-control", control_payload=control_payload) elif action == "stop_detection": self.request_stop_detection(source="redis-control") elif action: logger.warning(f"收到未知 Worker 控制动作: {action}") def connect_signals(self): """ 连接信号到槽函数 """ if self.main_window: try: # 连接信号到槽函数 self.config_signal.config_updated.connect(self.main_window.update_config_labels) logger.debug("信号连接成功") # 连接后立即更新一次配置 self.update_config_labels() except Exception as e: logger.error(f"连接信号失败: {e}") def _init_optional_detectors(self): if self.detect_options.get('detect_jucha') and self.jc_instance is None: self.jc_instance = jucha.JC(proxies=None) self.jc_instance.load_juming_cookies() self.jc_instance.load_cookies() if self.detect_options.get('detect_juziseo') and self.juziseo_instance is None: self.juziseo_instance = juziseo.Juziseo(proxies=None) self.juziseo_instance.load_cookies() def load_detect_options(self): """ 加载检测选项 """ default_order = [ 'detect_register', 'detect_baidu_site', 'detect_360_site', 'detect_chinaz', 'detect_aizhan', 'detect_wayback', 'detect_jucha', 'detect_juziseo', ] default_options = { 'detect_register': True, 'detect_wayback': True, 'detect_chinaz': True, 'detect_aizhan': True, 'detect_baidu_site': True, 'detect_360_site': True, 'detect_jucha': False, 'detect_juziseo': False, 'detect_order': default_order, } try: # 从Redis获取配置 if getattr(self, "use_redis", False): detect_options_str = self.redis_client.get('domain_tool:detect_options') if detect_options_str: detect_options = default_options.copy() detect_options.update(json.loads(detect_options_str)) if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'): detect_options['detect_jucha'] = True if detect_options.get('detect_juziseo_outlink'): detect_options['detect_juziseo'] = True order = detect_options.get('detect_order') or [] detect_options['detect_order'] = [key for key in order if key in default_order] for key in default_order: if key not in detect_options['detect_order']: detect_options['detect_order'].append(key) logger.info(f"从Redis加载检测选项成功: {detect_options}") return detect_options # 从本地文件获取配置 if os.path.exists('detect_options.json'): with open('detect_options.json', 'r', encoding='utf-8') as f: detect_options = default_options.copy() detect_options.update(json.load(f)) if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'): detect_options['detect_jucha'] = True if detect_options.get('detect_juziseo_outlink'): detect_options['detect_juziseo'] = True order = detect_options.get('detect_order') or [] detect_options['detect_order'] = [key for key in order if key in default_order] for key in default_order: if key not in detect_options['detect_order']: detect_options['detect_order'].append(key) logger.info(f"从本地文件加载检测选项成功: {detect_options}") return detect_options else: logger.info(f"使用默认检测选项: {default_options}") return default_options except Exception as e: logger.error(f"加载检测选项失败: {e}") logger.info(f"使用默认检测选项: {default_options}") return default_options def load_proxy_config(self): """ 加载代理配置 """ default_config = {'proxy_enable': False, 'proxy_url': '', 'proxy_urls': [], 'allow_direct': False} try: # 从Redis获取配置 if self.use_redis: proxy_config_str = self.redis_client.get('domain_tool:proxy_config') if proxy_config_str: proxy_config = default_config.copy() proxy_config.update(json.loads(proxy_config_str)) proxy_urls = proxy_config.get('proxy_urls') or [] if not proxy_urls and proxy_config.get('proxy_url'): proxy_urls = [proxy_config.get('proxy_url', '')] proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()] proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else '' proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False)) logger.info(f"从Redis加载代理配置成功: {proxy_config}") return proxy_config # 从本地文件获取配置 if os.path.exists('proxy_config.json'): with open('proxy_config.json', 'r', encoding='utf-8') as f: proxy_config = default_config.copy() proxy_config.update(json.load(f)) logger.info(f"从本地文件加载代理配置成功: {proxy_config}") proxy_urls = proxy_config.get('proxy_urls') or [] if not proxy_urls and proxy_config.get('proxy_url'): proxy_urls = [proxy_config.get('proxy_url', '')] proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()] proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else '' proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False)) return proxy_config else: logger.info(f"使用默认代理配置: {default_config}") return default_config except Exception as e: logger.error(f"加载代理配置失败: {e}") logger.info(f"使用默认代理配置: {default_config}") return default_config def _worker_parent_node_code(self): return str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() def _worker_node_code_candidates(self): candidates = [] for candidate in ( str(getattr(config, "NODE_CODE", "") or "").strip(), self._worker_parent_node_code(), ): if candidate and candidate not in candidates: candidates.append(candidate) return candidates def load_thread_count(self): """ 加载检测线程数 """ try: node_code_candidates = self._worker_node_code_candidates() node_code = node_code_candidates[0] if node_code_candidates else "" node_thread_counts = {} redis_global_thread_count = None def _warn_missing_node_override(source_name, overrides, fallback_value): if not node_code or not isinstance(overrides, dict) or not overrides: return for candidate in node_code_candidates: if candidate in overrides: return available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip())) logger.warning( "节点线程覆盖未命中，回退到%s通用线程数: node_code=%s, fallback=%s, available_node_codes=%s" % ( source_name, node_code, fallback_value, available_codes or "-", ) ) def _resolve_node_override(overrides, source_name): if not isinstance(overrides, dict): return None for candidate in node_code_candidates: node_thread_count_raw = overrides.get(candidate) if node_thread_count_raw is None: continue thread_count = max(1, int(node_thread_count_raw)) if candidate == node_code: logger.info(f"从{source_name}加载节点专属检测线程数成功: {candidate} -> {thread_count}") else: logger.info( f"从{source_name}加载父节点检测线程覆盖成功: {candidate} -> {thread_count} (current={node_code})" ) return thread_count return None # 从Redis获取配置 if self.use_redis: node_thread_counts_raw = self.redis_client.get('domain_tool:node_thread_counts') if node_thread_counts_raw: try: node_thread_counts = json.loads(node_thread_counts_raw) except Exception as e: logger.warning(f"解析 Redis 节点线程覆盖配置失败: {e}") resolved_node_thread_count = _resolve_node_override(node_thread_counts, "Redis") if resolved_node_thread_count is not None: return resolved_node_thread_count thread_count_str = self.redis_client.get('domain_tool:thread_count') if thread_count_str: thread_count = max(1, int(thread_count_str)) redis_global_thread_count = thread_count _warn_missing_node_override("Redis", node_thread_counts, thread_count) logger.info(f"从Redis加载检测线程数成功: {thread_count}") return thread_count # 从本地文件获取配置 node_thread_counts_payload = _read_worker_local_json_config('node_thread_counts.json') if isinstance(node_thread_counts_payload, dict): node_thread_counts = node_thread_counts_payload resolved_node_thread_count = _resolve_node_override(node_thread_counts, "本地文件") if resolved_node_thread_count is not None: return resolved_node_thread_count thread_config = _read_worker_local_json_config('thread_count.json') if isinstance(thread_config, dict): thread_count = thread_config.get('thread_count', '10') thread_count = max(1, int(thread_count)) if redis_global_thread_count is None: _warn_missing_node_override("本地文件", node_thread_counts, thread_count) logger.info(f"从本地文件加载检测线程数成功: {thread_count}") return thread_count else: default_thread_count = 1000 logger.info(f"使用默认检测线程数: {default_thread_count}") return default_thread_count except Exception as e: logger.error(f"加载检测线程数失败: {e}") default_thread_count = 1000 logger.info(f"使用默认检测线程数: {default_thread_count}") return default_thread_count def load_process_count(self): """ 加载检测进程数 """ try: node_code_candidates = self._worker_node_code_candidates() node_code = node_code_candidates[0] if node_code_candidates else "" node_process_counts = {} redis_global_process_count = None def _warn_missing_node_override(source_name, overrides, fallback_value): if not node_code or not isinstance(overrides, dict) or not overrides: return for candidate in node_code_candidates: if candidate in overrides: return available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip())) logger.warning( "节点进程覆盖未命中，回退到%s通用进程数: node_code=%s, fallback=%s, available_node_codes=%s" % ( source_name, node_code, fallback_value, available_codes or "-", ) ) def _resolve_node_override(overrides, source_name): if not isinstance(overrides, dict): return None for candidate in node_code_candidates: node_process_count_raw = overrides.get(candidate) if node_process_count_raw is None: continue process_count = max(1, int(node_process_count_raw)) if candidate == node_code: logger.info(f"从{source_name}加载节点专属检测进程数成功: {candidate} -> {process_count}") else: logger.info( f"从{source_name}加载父节点检测进程覆盖成功: {candidate} -> {process_count} (current={node_code})" ) return process_count return None if self.use_redis: node_process_counts_raw = self.redis_client.get('domain_tool:node_process_counts') if node_process_counts_raw: try: node_process_counts = json.loads(node_process_counts_raw) except Exception as e: logger.warning(f"解析 Redis 节点进程覆盖配置失败: {e}") resolved_node_process_count = _resolve_node_override(node_process_counts, "Redis") if resolved_node_process_count is not None: return resolved_node_process_count process_count_str = self.redis_client.get('domain_tool:process_count') if process_count_str: process_count = max(1, int(process_count_str)) redis_global_process_count = process_count _warn_missing_node_override("Redis", node_process_counts, process_count) logger.info(f"从Redis加载检测进程数成功: {process_count}") return process_count node_process_counts_payload = _read_worker_local_json_config('node_process_counts.json') if isinstance(node_process_counts_payload, dict): node_process_counts = node_process_counts_payload resolved_node_process_count = _resolve_node_override(node_process_counts, "本地文件") if resolved_node_process_count is not None: return resolved_node_process_count process_config = _read_worker_local_json_config('process_count.json') if isinstance(process_config, dict): process_count = process_config.get('process_count', '1') process_count = max(1, int(process_count)) if redis_global_process_count is None: _warn_missing_node_override("本地文件", node_process_counts, process_count) logger.info(f"从本地文件加载检测进程数成功: {process_count}") return process_count default_process_count = max( 1, int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1), ) logger.info(f"使用默认检测进程数: {default_process_count}") return default_process_count except Exception as e: logger.error(f"加载检测进程数失败: {e}") default_process_count = max( 1, int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1), ) logger.info(f"使用默认检测进程数: {default_process_count}") return default_process_count def refresh_thread_count_runtime(self, *, force=False, min_interval=2.0): """ 运行中热刷新线程数配置。 Web 后台改了节点并发后，不应该等整批任务跑完才生效。 """ now_ts = time.time() last_refresh_at = float(getattr(self, "_last_thread_count_refresh_at", 0.0) or 0.0) if not force and now_ts - last_refresh_at < max(0.2, float(min_interval or 0.0)): return max(1, int(getattr(self, "thread_count", 1) or 1)) previous = max(1, int(getattr(self, "thread_count", 1) or 1)) latest = max(1, int(self.load_thread_count() or previous)) self._last_thread_count_refresh_at = now_ts if latest != previous: self.thread_count = latest self.proxy_max_reuse_count = max(8, self.thread_count * 4) logger.info(f"检测线程数热更新: {previous} -> {latest}") self.update_config_labels() return max(1, int(getattr(self, "thread_count", latest) or latest)) def refresh_process_count_runtime(self, *, force=False, min_interval=5.0): """ 运行中热刷新进程数配置。 """ now_ts = time.time() last_refresh_at = float(getattr(self, "_last_process_count_refresh_at", 0.0) or 0.0) if not force and now_ts - last_refresh_at < max(0.5, float(min_interval or 0.0)): return max(1, int(getattr(self, "process_count", 1) or 1)) previous = max(1, int(getattr(self, "process_count", 1) or 1)) latest = max(1, int(self.load_process_count() or previous)) self._last_process_count_refresh_at = now_ts if latest != previous: self.process_count = latest logger.info(f"检测进程数热更新: {previous} -> {latest}") self.update_config_labels() return max(1, int(getattr(self, "process_count", latest) or latest)) def _proxy_runtime_process_count(self): current = max(1, int(getattr(self, "process_count", 1) or 1)) try: latest = self.refresh_process_count_runtime(min_interval=5.0) except Exception: latest = current return max(1, int(latest or current)) def _proxy_multi_process_divisor(self): process_count = self._proxy_runtime_process_count() if process_count <= 1: return 1.0 return max(1.0, math.sqrt(float(process_count))) def _proxy_refresh_holdoff_seconds(self): process_count = self._proxy_runtime_process_count() if process_count < 4 or not self.detecting: return 0.0 pid_spread = (os.getpid() % 7) * 0.02 return min(0.9, 0.12 + math.log2(float(process_count)) * 0.06 + pid_spread) def _proxy_coordination_node_code(self): return self._worker_parent_node_code() or str(getattr(config, "NODE_CODE", "") or "").strip() def _proxy_should_coordinate_shared_refresh(self): return bool( self.use_redis and self.redis_client is not None and self._proxy_runtime_process_count() > 1 and self._proxy_coordination_node_code() ) def _shared_proxy_pool_key(self): node_code = self._proxy_coordination_node_code() if not node_code: return "" return f"{SHARED_PROXY_POOL_KEY_PREFIX}:{node_code}" def _shared_proxy_refresh_lock_key(self): node_code = self._proxy_coordination_node_code() if not node_code: return "" return f"{SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX}:{node_code}" def _shared_proxy_snapshot_ttl_seconds(self): configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_SNAPSHOT_TTL", "18") or 18) return max(5, configured) def _shared_proxy_refresh_lock_seconds(self): configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REFRESH_LOCK_SECONDS", "8") or 8) return max(2, configured) def _shared_proxy_wait_timeout_seconds(self): configured = float(os.getenv("DOMAINCHECK_SHARED_PROXY_WAIT_TIMEOUT", "0.9") or 0.9) return max(0.2, configured) def _shared_proxy_reuse_gap(self): configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REUSE_GAP", "0") or 0) if configured > 0: return max(8, configured) refresh_threshold = max(1, int(self._proxy_refresh_threshold() or 1)) return max(24, min(160, max(24, refresh_threshold // 3))) def _shared_proxy_snapshot_should_replace_local_pool(self, shared_count, local_count, refresh_threshold): normalized_shared = max(0, int(shared_count or 0)) normalized_local = max(0, int(local_count or 0)) normalized_threshold = max(1, int(refresh_threshold or 1)) if normalized_shared <= 0: return False if normalized_local <= 0: return True if normalized_shared >= normalized_threshold: return True if normalized_shared <= normalized_local: return False return (normalized_shared - normalized_local) >= self._shared_proxy_reuse_gap() def _config_update_proxy_refresh_debounce_seconds(self): configured = float(os.getenv("DOMAINCHECK_PROXY_CONFIG_UPDATE_REFRESH_DEBOUNCE", "20") or 20) return max(2.0, configured) def _shared_proxy_snapshot_owner(self): return f"{config.NODE_CODE}:{os.getpid()}:{int(time.time() * 1000)}" def _rotate_proxy_entries_for_current_worker(self, proxy_entries): entries = list(proxy_entries or []) if len(entries) <= 1: return entries seed_source = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid()) seed = sum(ord(ch) for ch in seed_source) + int(os.getpid() or 0) offset = seed % len(entries) if offset <= 0: return entries return entries[offset:] + entries[:offset] def _shared_proxy_payload_from_entries(self, proxy_entries, *, source_count=0, raw_items=0): now_ts = time.time() shareable_entries = [] for proxy_entry in list(proxy_entries or []): if not isinstance(proxy_entry, dict) or not proxy_entry.get("proxy"): continue if not self._proxy_entry_can_be_reused(proxy_entry, now_ts=now_ts): continue shareable_entries.append(dict(proxy_entry)) if not shareable_entries: return None return { "node_code": self._proxy_coordination_node_code(), "owner": str(getattr(config, "NODE_CODE", "") or "").strip(), "refreshed_at_ts": now_ts, "source_count": int(source_count or 0), "raw_items": int(raw_items or 0), "available_count": len(shareable_entries), "proxy_pool": shareable_entries, } def _load_shared_proxy_snapshot_payload(self, *, max_age_seconds=None): if not self._proxy_should_coordinate_shared_refresh(): return None redis_key = self._shared_proxy_pool_key() if not redis_key: return None try: raw_payload = self.redis_client.get(redis_key) except Exception as exc: logger.debug(f"读取共享代理快照失败: {exc}") return None if not raw_payload: return None try: payload = json.loads(raw_payload) except Exception as exc: logger.debug(f"解析共享代理快照失败: {exc}") return None if not isinstance(payload, dict): return None refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0) allowed_age = max( 1.0, float(max_age_seconds if max_age_seconds is not None else self._shared_proxy_snapshot_ttl_seconds()), ) if refreshed_at_ts > 0 and (time.time() - refreshed_at_ts) > allowed_age: return None shareable_payload = self._shared_proxy_payload_from_entries( payload.get("proxy_pool") or [], source_count=payload.get("source_count", 0), raw_items=payload.get("raw_items", 0), ) if not shareable_payload: return None shareable_payload["owner"] = str(payload.get("owner") or "").strip() shareable_payload["refreshed_at_ts"] = refreshed_at_ts or shareable_payload.get("refreshed_at_ts", time.time()) return shareable_payload def _apply_shared_proxy_snapshot(self, payload, *, status_prefix="复用共享代理快照"): if not isinstance(payload, dict): return False proxy_entries = payload.get("proxy_pool") or [] if not isinstance(proxy_entries, list) or not proxy_entries: return False rotated_entries = self._rotate_proxy_entries_for_current_worker(proxy_entries) with self.proxy_pool_lock: self.proxy_pool = list(rotated_entries) pool_size = len(self.proxy_pool) refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0) self.proxy_last_refresh_time = ( datetime.fromtimestamp(refreshed_at_ts) if refreshed_at_ts > 0 else datetime.now() ) self.proxy_last_refresh_source_count = int(payload.get("source_count", 0) or 0) self.proxy_last_refresh_total_items = int(payload.get("raw_items", 0) or 0) self.proxy_last_validated_count = 0 self.proxy_last_available_count = pool_size source_owner = str(payload.get("owner") or "").strip() if source_owner and source_owner != str(getattr(config, "NODE_CODE", "") or "").strip(): self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个（来源 {source_owner}）" else: self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个" return True def _publish_shared_proxy_snapshot(self, proxy_entries, *, source_count=0, raw_items=0): if not self._proxy_should_coordinate_shared_refresh(): return False payload = self._shared_proxy_payload_from_entries( proxy_entries, source_count=source_count, raw_items=raw_items, ) if not payload: return False redis_key = self._shared_proxy_pool_key() if not redis_key: return False try: self.redis_client.set( redis_key, json.dumps(payload, ensure_ascii=False), ex=self._shared_proxy_snapshot_ttl_seconds(), ) return True except Exception as exc: logger.debug(f"写入共享代理快照失败: {exc}") return False def _acquire_shared_proxy_refresh_lock(self): if not self._proxy_should_coordinate_shared_refresh(): return "" redis_key = self._shared_proxy_refresh_lock_key() if not redis_key: return "" owner_token = self._shared_proxy_snapshot_owner() try: acquired = self.redis_client.set( redis_key, owner_token, nx=True, ex=self._shared_proxy_refresh_lock_seconds(), ) except Exception as exc: logger.debug(f"获取共享代理刷新锁失败: {exc}") return "" return owner_token if acquired else "" def _release_shared_proxy_refresh_lock(self, owner_token): if not owner_token or not self._proxy_should_coordinate_shared_refresh(): return redis_key = self._shared_proxy_refresh_lock_key() if not redis_key: return try: current_owner = self.redis_client.get(redis_key) if str(current_owner or "") == str(owner_token): self.redis_client.delete(redis_key) except Exception as exc: logger.debug(f"释放共享代理刷新锁失败: {exc}") def _wait_for_shared_proxy_snapshot(self, timeout_seconds=None): if not self._proxy_should_coordinate_shared_refresh(): return False deadline = time.time() + max( 0.2, float(timeout_seconds if timeout_seconds is not None else self._shared_proxy_wait_timeout_seconds()), ) while time.time() < deadline: payload = self._load_shared_proxy_snapshot_payload() if payload and self._apply_shared_proxy_snapshot(payload): return True time.sleep(0.05) return False def _should_trigger_proxy_refresh_for_config_update( self, *, config_type="", previous_proxy_config=None, previous_thread_count=0, current_thread_count=0, ): previous_proxy_config = dict(previous_proxy_config or {}) previous_urls = tuple(previous_proxy_config.get('proxy_urls') or []) current_urls = tuple((self.proxy_config or {}).get('proxy_urls') or []) proxy_changed = ( bool(previous_proxy_config.get('proxy_enable', False)) != bool((self.proxy_config or {}).get('proxy_enable', False)) or bool(previous_proxy_config.get('allow_direct', False)) != bool((self.proxy_config or {}).get('allow_direct', False)) or previous_urls != current_urls ) if proxy_changed: return True, True normalized_config_type = str(config_type or "").strip() if normalized_config_type not in {"thread_count", "node_thread_counts"}: return False, False if int(current_thread_count or 0) == int(previous_thread_count or 0): return False, False with self.proxy_pool_lock: local_available = len(self.proxy_pool) shared_payload = self._load_shared_proxy_snapshot_payload() shared_available = 0 if isinstance(shared_payload, dict): shared_available = len(list(shared_payload.get("proxy_pool") or [])) effective_available = max(local_available, shared_available) refresh_threshold = self._proxy_refresh_threshold() if effective_available >= max(1, int(refresh_threshold or 1)): return False, False shortage = max(0, int(refresh_threshold or 0) - effective_available) if effective_available <= 0: return True, False return shortage >= self._shared_proxy_reuse_gap(), False def _pull_sync_tasks_until_available(self, *, thread_limit: int, claim_after_pull: bool = True) -> list[dict]: """ mainland-controller 在本地任务吃空时，不要立即退出检测循环，而是主动向海外控制面 连续补货几轮，尽量把本地队列重新喂满。 """ if str(getattr(config, "NODE_REGION", "") or "").strip() != "mainland": return [] if str(getattr(config, "NODE_ROLE", "") or "").strip() != "control": return [] max_rounds = max(1, int(os.getenv("DOMAINCHECK_SYNC_PULL_BURST_ROUNDS", "4") or 4)) current_thread_limit = max(1, int(thread_limit or getattr(self, "thread_count", 1) or 1)) pull_limit = self._resolve_sync_pull_limit(current_thread_limit) claim_batch_size = max(20, current_thread_limit) claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) maintenance_lock_owner = self._acquire_shared_job_maintenance_lock( "pull-sync-tasks", ttl_seconds=max(15, max_rounds * 12), ) if not maintenance_lock_owner: return [] local_api_base_url = str( os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1" ).strip().rstrip("/") try: for round_index in range(1, max_rounds + 1): try: request = urllib.request.Request( f"{local_api_base_url}/runtime/actions/pull_tasks", data=json.dumps({"limit": pull_limit}, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=max(15, min(60, 10 + pull_limit // 100))) as response: raw = response.read().decode("utf-8") response_data = json.loads(raw) if raw else {} pull_ok = int(response_data.get("code", 1) or 1) == 0 pull_message = str(response_data.get("message") or "").strip() pull_data = response_data.get("data") or {} except Exception as exc: logger.warning(f"本地队列补货失败: round={round_index}/{max_rounds}, error={exc}") self._sync_worker_log_event( "本地队列补货失败", level='warning', payload={ "round": round_index, "max_rounds": max_rounds, "pull_limit": pull_limit, "error": str(exc), }, mode='full', ) break pull_state = str((pull_data or {}).get("pull_state") or "").strip() queued_count = int((pull_data or {}).get("queued_count", 0) or 0) worker_start_message = str((pull_data or {}).get("worker_start_message") or "").strip() refill_message = ( f"本地队列主动补货: round={round_index}/{max_rounds} | " f"pull_limit={pull_limit} | ok={1 if pull_ok else 0} | " f"state={pull_state or 'unknown'} | queued_count={queued_count}" ) if worker_start_message: refill_message = f"{refill_message} | worker_start={worker_start_message}" logger.info(f"{refill_message} | message={pull_message}") self._sync_worker_log_event( refill_message, payload={ "round": round_index, "max_rounds": max_rounds, "pull_limit": pull_limit, "pull_ok": bool(pull_ok), "pull_state": pull_state, "queued_count": queued_count, "message": str(pull_message or "").strip(), "worker_start_message": worker_start_message, }, mode='full', ) if not claim_after_pull: if queued_count > 0: logger.info( f"主动补货已写入本地队列: round={round_index}/{max_rounds}, " f"queued_count={queued_count}, claim_after_pull=0" ) return [] else: domains = self._claim_detect_job_items( limit=claim_batch_size, lease_seconds=claim_lease_seconds, ) if domains: logger.info( f"主动补货后已重新领取任务: round={round_index}/{max_rounds}, " f"claim_batch_size={claim_batch_size}, domains={len(domains)}" ) return domains if pull_state == "idle": break if not pull_ok and pull_state not in {"ack_warning", "worker_start_warning"}: break time.sleep(0.2) return [] finally: self._release_shared_job_maintenance_lock("pull-sync-tasks", maintenance_lock_owner) def _resolve_sync_pull_limit(self, current_thread_limit: int) -> int: configured_pull_limit = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT", "0") or 0) configured_pull_cap = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT_CAP", "200000") or 200000) pull_cap = max(10000, configured_pull_cap) if configured_pull_limit > 0: return max(1000, min(configured_pull_limit, pull_cap)) normalized_thread_limit = max(1, int(current_thread_limit or getattr(self, "thread_count", 1) or 1)) process_count = max(1, int(self._proxy_runtime_process_count() or 1)) total_thread_capacity = max( normalized_thread_limit, normalized_thread_limit * process_count, ) default_pull_limit = max( 4000, min(pull_cap, total_thread_capacity * 2), ) return max(1000, int(default_pull_limit or 1000)) def _process_pipeline_tasks_until_available(self, *, thread_limit: int) -> list[dict]: """ 当本地任务队列空了时，主动让 controller 处理已完成步骤，尽量把下一步任务补出来。 """ maintenance_lock_owner = self._acquire_shared_job_maintenance_lock( "process-pipeline-tasks", ttl_seconds=20, ) if not maintenance_lock_owner: return [] local_api_base_url = str( os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1" ).strip().rstrip("/") current_thread_limit = max(1, int(thread_limit or getattr(self, "thread_count", 1) or 1)) configured_process_limit = int(os.getenv("DOMAINCHECK_PIPELINE_PROCESS_LIMIT", "0") or 0) process_limit = max( 20, configured_process_limit if configured_process_limit > 0 else current_thread_limit * 4, ) claim_batch_size = max(20, current_thread_limit) claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) try: try: request = urllib.request.Request( f"{local_api_base_url}/runtime/actions/process_pipeline", data=json.dumps({"limit": process_limit}, ensure_ascii=False).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=20) as response: raw = response.read().decode("utf-8") response_data = json.loads(raw) if raw else {} process_ok = int(response_data.get("code", 1) or 1) == 0 process_message = str(response_data.get("message") or "").strip() process_data = response_data.get("data") or {} logger.info( f"本地 pipeline 推进: ok={1 if process_ok else 0} | " f"processed={int(process_data.get('processed_items', 0) or 0)} | " f"advanced={int(process_data.get('advanced_items', 0) or 0)} | " f"retried={int(process_data.get('retried_items', 0) or 0)} | " f"message={process_message}" ) except Exception as exc: logger.warning(f"本地 pipeline 推进失败: {exc}") self._sync_worker_log_event( "本地 pipeline 推进失败", level='warning', payload={"error": str(exc), "process_limit": process_limit}, mode='full', ) return [] domains = self._claim_detect_job_items( limit=claim_batch_size, lease_seconds=claim_lease_seconds, ) if domains: self._sync_worker_log_event( "本地 pipeline 推进后已重新领取任务", payload={"domains": len(domains), "claim_batch_size": claim_batch_size}, mode='full', ) return domains finally: self._release_shared_job_maintenance_lock("process-pipeline-tasks", maintenance_lock_owner) def load_runtime_settings(self): default_settings = { "worker_log_sync_enabled": False, "worker_log_sync_mode": "key", "worker_step_trace_enabled": True, "worker_step_trace_sync_full": True, "control_node_autoresume_enabled": False, "claim_recent_jobs_first": False, "claim_recent_jobs_limit": 0, "claim_recent_jobs_window_hours": 0, "claim_batch_floor": 0, "claim_batch_ceil": 0, "submit_backlog_floor": 0, "submit_backlog_ceil": 0, "dispatch_cap_multiplier": 1, "pending_buffer_cap_multiplier": 1, } def _normalize_bool(value, default=False): if value is None: return bool(default) if isinstance(value, bool): return value return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"} def _normalize_non_negative_int(value, default=0): try: normalized = int(value) except (TypeError, ValueError): normalized = int(default) return max(0, normalized) def _normalize_positive_int(value, default=1): try: normalized = int(value) except (TypeError, ValueError): normalized = int(default) return max(1, normalized) def _normalize_runtime_settings(payload): runtime_settings = default_settings.copy() runtime_settings.update(payload or {}) runtime_settings["worker_log_sync_enabled"] = _normalize_bool(runtime_settings.get("worker_log_sync_enabled", False)) runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" runtime_settings["worker_step_trace_enabled"] = _normalize_bool(runtime_settings.get("worker_step_trace_enabled", True), default=True) runtime_settings["worker_step_trace_sync_full"] = _normalize_bool(runtime_settings.get("worker_step_trace_sync_full", True), default=True) runtime_settings["control_node_autoresume_enabled"] = _normalize_bool( runtime_settings.get("control_node_autoresume_enabled", False), default=False, ) runtime_settings["claim_recent_jobs_first"] = _normalize_bool( runtime_settings.get("claim_recent_jobs_first", False), default=False, ) for key in ( "claim_recent_jobs_limit", "claim_recent_jobs_window_hours", "claim_batch_floor", "claim_batch_ceil", "submit_backlog_floor", "submit_backlog_ceil", ): runtime_settings[key] = _normalize_non_negative_int(runtime_settings.get(key), default_settings[key]) for key in ("dispatch_cap_multiplier", "pending_buffer_cap_multiplier"): runtime_settings[key] = _normalize_positive_int(runtime_settings.get(key), default_settings[key]) return runtime_settings try: if self.use_redis: runtime_settings_raw = self.redis_client.get(RUNTIME_SETTINGS_KEY) if runtime_settings_raw: runtime_settings = _normalize_runtime_settings(json.loads(runtime_settings_raw)) logger.info(f"从Redis加载运行时设置成功: {runtime_settings}") return runtime_settings candidate_paths = [ 'runtime_settings.json', os.path.join('runtime', 'runtime_settings.json'), os.path.join('..', 'domain-api', 'runtime', 'runtime_settings.json'), ] for candidate_path in candidate_paths: if os.path.exists(candidate_path): with open(candidate_path, 'r', encoding='utf-8') as f: runtime_settings = _normalize_runtime_settings(json.load(f)) logger.info(f"从本地文件加载运行时设置成功: {runtime_settings}") return runtime_settings except Exception as e: logger.error(f"加载运行时设置失败: {e}") return default_settings def _runtime_optional_int_override(self, runtime_key, env_key): runtime_settings = self.runtime_settings or {} try: runtime_value = int(runtime_settings.get(runtime_key, 0) or 0) except (TypeError, ValueError): runtime_value = 0 if runtime_value > 0: return runtime_value try: return max(0, int(os.getenv(env_key, "0") or 0)) except (TypeError, ValueError): return 0 def _runtime_bool_override(self, runtime_key, env_key, default=False): env_value = os.getenv(env_key) if env_value is not None: return self._normalize_flag_value(env_value, default=default) runtime_settings = self.runtime_settings or {} if runtime_key in runtime_settings: return self._normalize_flag_value(runtime_settings.get(runtime_key, default), default=default) return bool(default) def _runtime_positive_int_override(self, runtime_key, env_key, default=1): runtime_settings = self.runtime_settings or {} try: runtime_value = int(runtime_settings.get(runtime_key, 0) or 0) except (TypeError, ValueError): runtime_value = 0 if runtime_value > 0: return runtime_value try: return max(1, int(os.getenv(env_key, str(default)) or default)) except (TypeError, ValueError): return max(1, int(default)) def _claim_recent_jobs_first_enabled(self): return self._runtime_bool_override( "claim_recent_jobs_first", "DOMAINCHECK_CLAIM_RECENT_JOBS_FIRST", default=False, ) def _preferred_claim_job_limit(self): return max(1, self._runtime_optional_int_override( "claim_recent_jobs_limit", "DOMAINCHECK_CLAIM_RECENT_JOBS_LIMIT", ) or 8) def _preferred_claim_job_window_hours(self): return max(1, self._runtime_optional_int_override( "claim_recent_jobs_window_hours", "DOMAINCHECK_CLAIM_RECENT_JOBS_WINDOW_HOURS", ) or 24) def _claim_detect_job_items(self, *, limit, lease_seconds, job_id=None): if job_id in (None, "", 0, "0") and not self._is_single_step_session_active(): handoff_job_id = self._restart_release_handoff_job_id_active() if handoff_job_id: if not self._is_primary_job_maintenance_worker(): return [] handoff_limit = self._restart_release_handoff_batch_limit(limit) handoff_rows = self.db.claim_restart_released_detect_job_items( config.NODE_CODE, handoff_job_id, limit=handoff_limit, lease_seconds=lease_seconds, ) if handoff_rows: logger.warning( f"优先回收重启尾批任务成功: job_id={handoff_job_id}, " f"claimed={len(handoff_rows)}, reason={self._restart_release_handoff_reason or 'worker_restart'}" ) remaining = max(0, int(limit or 0) - len(handoff_rows)) if remaining <= 0: return handoff_rows prefer_recent_jobs = self._claim_recent_jobs_first_enabled() fallback_rows = self.db.claim_detect_job_items( config.NODE_CODE, limit=remaining, lease_seconds=lease_seconds, job_id=None, prefer_recent_jobs=prefer_recent_jobs, preferred_recent_job_limit=self._preferred_claim_job_limit(), preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), ) return list(handoff_rows or []) + list(fallback_rows or []) self._clear_restart_release_handoff(reason="handoff_queue_drained") explicit_scope_job_id = self._explicit_claim_scope_job_id_active() if explicit_scope_job_id not in (None, "", 0, "0"): scoped_rows = self.db.claim_detect_job_items( config.NODE_CODE, limit=limit, lease_seconds=lease_seconds, job_id=explicit_scope_job_id, prefer_recent_jobs=False, preferred_recent_job_limit=self._preferred_claim_job_limit(), preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), ) self._log_explicit_scope_claim_result( scoped_job_id=explicit_scope_job_id, limit=limit, rows_count=len(scoped_rows or []), reason="explicit_scope", ) return scoped_rows prefer_recent_jobs = bool(job_id in (None, "", 0, "0") and self._claim_recent_jobs_first_enabled()) return self.db.claim_detect_job_items( config.NODE_CODE, limit=limit, lease_seconds=lease_seconds, job_id=job_id, prefer_recent_jobs=prefer_recent_jobs, preferred_recent_job_limit=self._preferred_claim_job_limit(), preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), ) def _load_sensitive_words_runtime(self): try: if self.use_redis and self.redis_client is not None: sensitive_words_raw = self.redis_client.get("domain_tool:sensitive_words") if sensitive_words_raw is not None: try: parsed = json.loads(sensitive_words_raw) except Exception: parsed = sensitive_words_raw if isinstance(parsed, list): words = [str(item).strip() for item in parsed if str(item).strip()] else: words = [line.strip() for line in str(parsed or "").splitlines() if line.strip()] logger.info(f"从Redis加载敏感词成功，共 {len(words)} 个敏感词") return words candidate_paths = [ 'runtime/sensitive_words.json', 'sensitive_words.json', 'sensitive_words.txt', ] for candidate_path in candidate_paths: if not os.path.exists(candidate_path): continue if candidate_path.endswith('.json'): with open(candidate_path, 'r', encoding='utf-8') as f: payload = json.load(f) if isinstance(payload, dict): if isinstance(payload.get('items'), list): words = [] for item in payload.get('items') or []: if isinstance(item, dict): word = str(item.get('word') or '').strip() else: word = str(item).strip() if word: words.append(word) else: words = [line.strip() for line in str(payload.get('text') or '').splitlines() if line.strip()] elif isinstance(payload, list): words = [str(item).strip() for item in payload if str(item).strip()] else: words = [] else: with open(candidate_path, 'r', encoding='utf-8') as f: words = [line.strip() for line in f.read().splitlines() if line.strip()] logger.info(f"从本地文件加载敏感词成功: {candidate_path}，共 {len(words)} 个敏感词") return words except Exception as e: logger.error(f"从运行时配置加载敏感词失败: {e}") words = self.db.get_all_sensitive_words() logger.info(f"从数据库加载敏感词成功，共 {len(words)} 个敏感词") return words @staticmethod def _remote_debug_ingest_url(): base_url = str(os.getenv("SYNC_TARGET_API_BASE_URL", "") or "").strip().rstrip("/") if not base_url: return "" if base_url.endswith("/api/v1"): return f"{base_url}/runtime/debug-ingest" if base_url.endswith("/api/v1/runtime"): return f"{base_url}/debug-ingest" return f"{base_url}/api/v1/runtime/debug-ingest" @staticmethod def _remote_debug_event_timeout_seconds(): configured_timeout = float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_TIMEOUT", "0") or 0.0) if configured_timeout > 0: return max(0.2, min(configured_timeout, 10.0)) if str(getattr(config, "NODE_ROLE", "") or "").strip() == "control": return 1.2 return 2.0 @staticmethod def _remote_debug_event_cooldown_seconds(failure_streak): normalized_streak = max(1, int(failure_streak or 1)) base_seconds = max( 5.0, float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_BASE_SECONDS", "15") or 15.0), ) max_seconds = max( base_seconds, float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_MAX_SECONDS", "180") or 180.0), ) return min(max_seconds, base_seconds * (2 ** max(0, normalized_streak - 1))) def _push_remote_debug_event(self, *, message, level='info', payload=None, event_type='worker_log'): ingest_url = self._remote_debug_ingest_url() if not ingest_url: return now_ts = time.time() cooldown_until = float(getattr(self, "_remote_debug_event_cooldown_until", 0.0) or 0.0) if cooldown_until > now_ts: return request_payload = { "source_region": config.NODE_REGION, "node_code": config.NODE_CODE, "hostname": socket.gethostname(), "service": "worker-event", "event_type": str(event_type or "worker_log"), "level": str(level or "info"), "message": str(message or "").strip()[:2000], "payload": payload or {}, } headers = { "Content-Type": "application/json", } shared_token = str(os.getenv("SYNC_SHARED_TOKEN", "") or "").strip() if shared_token: headers["X-Domaincheck-Sync-Token"] = shared_token request = urllib.request.Request( ingest_url, data=json.dumps(request_payload, ensure_ascii=False, sort_keys=True).encode("utf-8"), headers=headers, method="POST", ) try: timeout_seconds = self._remote_debug_event_timeout_seconds() with urllib.request.urlopen(request, timeout=timeout_seconds) as response: raw = response.read().decode("utf-8", errors="ignore") if raw: parsed = json.loads(raw) if isinstance(parsed, dict) and parsed.get("code") not in (0, "0", None, ""): logger.debug(f"远端调试事件返回非成功: {parsed}") self._remote_debug_event_failure_streak = 0 self._remote_debug_event_cooldown_until = 0.0 except Exception as e: failure_streak = int(getattr(self, "_remote_debug_event_failure_streak", 0) or 0) + 1 cooldown_seconds = self._remote_debug_event_cooldown_seconds(failure_streak) self._remote_debug_event_failure_streak = failure_streak self._remote_debug_event_cooldown_until = now_ts + cooldown_seconds last_notice_at = float(getattr(self, "_last_remote_debug_event_notice_at", 0.0) or 0.0) if failure_streak <= 2 or now_ts - last_notice_at >= max(10.0, min(cooldown_seconds, 30.0)): logger.debug( f"远端调试事件回传失败，进入冷却 {int(cooldown_seconds)} 秒: {e}" ) self._last_remote_debug_event_notice_at = now_ts def _worker_log_sync_mode(self): if not bool((self.runtime_settings or {}).get("worker_log_sync_enabled", False)): return "off" configured_mode = ( "full" if str((self.runtime_settings or {}).get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" ) if configured_mode != "full": return configured_mode degrade_reason = self._worker_log_sync_full_degrade_reason() if not degrade_reason: return configured_mode now_ts = time.time() last_notice_at = float(getattr(self, "_last_worker_log_sync_degrade_notice_at", 0.0) or 0.0) if now_ts - last_notice_at >= 10.0: logger.warning(f"Worker 全量日志同步已自动降级为 key: {degrade_reason}") self._last_worker_log_sync_degrade_notice_at = now_ts return "key" def _worker_log_sync_full_active_thread_threshold(self): configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_ACTIVE_THREADS", "0") or 0) if configured_threshold > 0: return max(32, configured_threshold) thread_limit = max(1, int(getattr(self, "thread_count", 0) or 0)) return max(120, min(600, max(120, thread_limit // 3))) def _worker_log_sync_full_queue_threshold(self): configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_QUEUE", "0") or 0) if configured_threshold > 0: return max(256, configured_threshold) queue_maxsize = int(getattr(getattr(self, "_worker_log_sync_queue", None), "maxsize", 0) or 0) if queue_maxsize > 0: return max(1000, min(15000, queue_maxsize // 5)) return 5000 def _worker_log_sync_full_degrade_reason(self): queue_obj = getattr(self, "_worker_log_sync_queue", None) queue_depth = 0 if queue_obj is not None: try: queue_depth = max(0, int(queue_obj.qsize() or 0)) except Exception: queue_depth = 0 queue_threshold = self._worker_log_sync_full_queue_threshold() if queue_depth >= queue_threshold: return f"queue_backlog={queue_depth}, threshold={queue_threshold}" live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) active_threshold = self._worker_log_sync_full_active_thread_threshold() if bool(getattr(self, "detecting", False)) and live_active_threads >= active_threshold: return f"active_threads={live_active_threads}, threshold={active_threshold}" return "" def _worker_step_trace_enabled(self): return bool((self.runtime_settings or {}).get("worker_step_trace_enabled", True)) def _worker_step_trace_sync_full_enabled(self): return bool((self.runtime_settings or {}).get("worker_step_trace_sync_full", True)) @staticmethod def _single_step_cache_key(domain_id, field_name): return f"domain_tool:single_step_result:{int(domain_id or 0)}:{str(field_name or '').strip()}" def _set_current_task_context(self, *, domain_id, is_step_task, task_mode, job_item_id=None): self._task_local.domain_id = int(domain_id or 0) self._task_local.is_step_task = bool(is_step_task) self._task_local.task_mode = str(task_mode or "").strip() self._task_local.job_item_id = int(job_item_id or 0) if job_item_id else None self._task_local.detect_status = None self._task_local.step_proxy_retry_waits = {} self._task_local.step_force_direct_once = {} def _clear_current_task_context(self): for attr in ("domain_id", "is_step_task", "task_mode", "job_item_id", "detect_status", "step_proxy_retry_waits", "step_force_direct_once"): try: delattr(self._task_local, attr) except Exception: pass def _is_current_single_step_task(self): return bool(getattr(self._task_local, "is_step_task", False)) def _set_task_detect_status(self, status): try: normalized_status = int(status) except Exception: normalized_status = None self._task_local.detect_status = normalized_status def _get_task_detect_status(self): try: value = getattr(self._task_local, "detect_status", None) return int(value) if value is not None else None except Exception: return None def _resolve_current_task_final_status(self): current_status = self._get_task_detect_status() if current_status == DETECT_STATUS_BLACKLISTED: return 'blacklisted' return 'failed' def _set_step_proxy_retry_wait(self, step_name, wait_seconds): normalized_step_name = str(step_name or "").strip() if not normalized_step_name: return try: normalized_wait_seconds = max(0.0, float(wait_seconds or 0.0)) except Exception: normalized_wait_seconds = 0.0 retry_waits = getattr(self._task_local, "step_proxy_retry_waits", None) if not isinstance(retry_waits, dict): retry_waits = {} self._task_local.step_proxy_retry_waits = retry_waits retry_waits[normalized_step_name] = normalized_wait_seconds def _consume_step_proxy_retry_wait(self, step_name): normalized_step_name = str(step_name or "").strip() if not normalized_step_name: return 0.0 retry_waits = getattr(self._task_local, "step_proxy_retry_waits", None) if not isinstance(retry_waits, dict): return 0.0 try: return max(0.0, float(retry_waits.pop(normalized_step_name, 0.0) or 0.0)) except Exception: return 0.0 def _set_step_force_direct_once(self, step_name, enabled=True): normalized_step_name = str(step_name or "").strip() if not normalized_step_name: return force_map = getattr(self._task_local, "step_force_direct_once", None) if not isinstance(force_map, dict): force_map = {} self._task_local.step_force_direct_once = force_map force_map[normalized_step_name] = bool(enabled) def _consume_step_force_direct_once(self, step_name): normalized_step_name = str(step_name or "").strip() if not normalized_step_name: return False force_map = getattr(self._task_local, "step_force_direct_once", None) if not isinstance(force_map, dict): return False return bool(force_map.pop(normalized_step_name, False)) def _peek_step_force_direct_once(self, step_name): normalized_step_name = str(step_name or "").strip() if not normalized_step_name: return False force_map = getattr(self._task_local, "step_force_direct_once", None) if not isinstance(force_map, dict): return False return bool(force_map.get(normalized_step_name, False)) def _cache_single_step_result_payload(self, domain_id, field_name, payload): normalized_field_name = str(field_name or "").strip() if not normalized_field_name: return cache_key = self._single_step_cache_key(domain_id, normalized_field_name) with self._step_result_cache_lock: self._step_result_cache[(int(domain_id or 0), normalized_field_name)] = dict(payload or {}) if self.use_redis and self.redis_client is not None: try: self.redis_client.setex(cache_key, 600, json.dumps(payload or {}, ensure_ascii=False)) except Exception as e: logger.debug(f"写入 Redis single_step 结果缓存失败: {e}") def _load_cached_single_step_result_payload(self, domain_id, field_name): normalized_field_name = str(field_name or "").strip() if not normalized_field_name: return None cache_tuple_key = (int(domain_id or 0), normalized_field_name) with self._step_result_cache_lock: cached_payload = self._step_result_cache.pop(cache_tuple_key, None) if isinstance(cached_payload, dict) and cached_payload: return dict(cached_payload) if self.use_redis and self.redis_client is not None: try: redis_key = self._single_step_cache_key(domain_id, normalized_field_name) raw_payload = self.redis_client.get(redis_key) if raw_payload: self.redis_client.delete(redis_key) parsed = json.loads(raw_payload) if isinstance(parsed, dict): return parsed except Exception as e: logger.debug(f"读取 Redis single_step 结果缓存失败: {e}") return None def _sync_worker_log_event(self, message, level='info', payload=None, *, mode='key', job_item_id=None): current_mode = self._worker_log_sync_mode() if current_mode == "off": return if mode == 'full' and current_mode != "full": return normalized_message = str(message or '').strip() if not normalized_message: return if normalized_message == self._last_synced_worker_log and mode != 'full': return event_payload = dict(payload or {}) explicit_job_id = event_payload.get("job_id") try: job_id = int(explicit_job_id) if explicit_job_id not in (None, "", 0, "0") else self.current_job_id except Exception: job_id = self.current_job_id if not job_id: return if self.current_cycle_token and not event_payload.get("cycle_token"): event_payload["cycle_token"] = self.current_cycle_token if self.current_job_code and not event_payload.get("job_code"): event_payload["job_code"] = self.current_job_code event_payload["log_mode"] = mode event_payload["synced_by"] = "worker_log_callback" try: queue_item = { "job_id": job_id, "job_item_id": job_item_id, "message": normalized_message, "level": level, "payload": event_payload, "write_db": not (mode == 'full' and self._is_current_single_step_task()), } try: self._worker_log_sync_queue.put_nowait(queue_item) except Full: if mode != 'full': self._worker_log_sync_queue.put(queue_item, timeout=0.2) else: raise self._last_synced_worker_log = normalized_message except Exception as e: if mode == 'full': self._worker_log_sync_drop_count += 1 now_ts = time.time() if now_ts - self._last_worker_log_sync_drop_notice_at >= 10: logger.warning( f"Worker 全量日志同步队列拥堵，已丢弃 {self._worker_log_sync_drop_count} 条 full 日志" ) self._last_worker_log_sync_drop_notice_at = now_ts else: logger.debug(f"回传Worker日志事件失败: {e}") @staticmethod def _step_display_name(detect_key): mapping = { 'detect_register': '注册状态检测', 'detect_baidu_site': '百度site检测', 'detect_360_site': '360 site检测', 'detect_chinaz': '站长之家检测', 'detect_aizhan': '爱站检测', 'detect_wayback': '时光机检测', 'detect_jucha': '聚查检测', 'detect_juziseo': '桔子检测', } return mapping.get(str(detect_key or '').strip(), str(detect_key or '').strip() or 'unknown') @staticmethod def _step_result_field_name(detect_key): mapping = { 'detect_register': 'register_status', 'detect_baidu_site': 'baidu_site', 'detect_360_site': 'qihu360_site', 'detect_chinaz': 'chinaz_info', 'detect_aizhan': 'aizhan_info', 'detect_wayback': 'wayback_info', } return mapping.get(str(detect_key or '').strip(), '') def _build_single_step_job_result(self, *, domain_id, detect_key, step_ok, step_name): field_name = self._step_result_field_name(detect_key) cached_payload = self._load_cached_single_step_result_payload(domain_id, field_name) if isinstance(cached_payload, dict) and cached_payload: payload = dict(cached_payload) payload["step_code"] = str(detect_key or "").strip() payload["step_name"] = str(step_name or "").strip() payload["field_name"] = field_name payload["task_mode"] = "single_step" payload["domain_id"] = int(domain_id or 0) if payload.get("state") == "degraded": payload["retry_recommended"] = True return payload try: domain_snapshot = self.db.get_domain_by_id(domain_id) or {} except Exception: domain_snapshot = {} raw_payload = domain_snapshot.get(field_name) if field_name else None payload = dict(raw_payload) if isinstance(raw_payload, dict) else {} if not payload: payload = { "status": bool(step_ok), "state": "passed" if step_ok else "failed", "message": "success" if step_ok else f"{step_name} 未通过", } payload["step_code"] = str(detect_key or "").strip() payload["step_name"] = str(step_name or "").strip() payload["field_name"] = field_name payload["task_mode"] = "single_step" payload["domain_id"] = int(domain_id or 0) if payload.get("state") == "degraded": payload["retry_recommended"] = True return payload @staticmethod def _resolve_single_step_finalization(result_payload): payload = dict(result_payload or {}) state = str(payload.get("state") or "").strip().lower() message = str(payload.get("message") or "").strip() status = payload.get("status") if state == "blacklisted": return "blacklisted", message or "步骤命中黑名单" if state == "degraded": return "completed", message or "步骤外部依赖异常，建议重试" if state in {"failed", "error", "rejected"}: return "failed", message or "步骤执行失败" if status is False: return "failed", message or "步骤执行失败" return "completed", message or "步骤执行完成" def _emit_step_trace(self, domain_name, step_name, stage, *, elapsed_ms=None, ok=None, level='info', job_item_id=None, sync_mode=None, **extra): if not self._worker_step_trace_enabled(): return payload = { "domain": str(domain_name or "").strip(), "step": str(step_name or "").strip(), "stage": str(stage or "").strip(), } if elapsed_ms is not None: payload["elapsed_ms"] = int(elapsed_ms or 0) if ok is not None: payload["ok"] = bool(ok) for key, value in dict(extra or {}).items(): if value in (None, ""): continue payload[key] = value message_parts = [ f"domain={payload.get('domain') or '-'}", f"step={payload.get('step') or '-'}", f"stage={payload.get('stage') or '-'}", ] if "elapsed_ms" in payload: message_parts.append(f"elapsed_ms={payload['elapsed_ms']}") if "ok" in payload: message_parts.append(f"ok={1 if payload['ok'] else 0}") for key in sorted(payload.keys()): if key in {"domain", "step", "stage", "elapsed_ms", "ok"}: continue value = payload.get(key) if isinstance(value, (dict, list, tuple)): value = json.dumps(value, ensure_ascii=False, sort_keys=True) message_parts.append(f"{key}={value}") message = "检测步骤跟踪: " + " | ".join(message_parts) logger.log(level.upper(), message) should_sync_full = self._worker_step_trace_sync_full_enabled() if sync_mode == 'off': should_sync_full = False if should_sync_full: self._sync_worker_log_event( message, level=level, payload=payload, mode='full', job_item_id=job_item_id, ) def test_proxy(self, proxy_item, result_queue): """ 测试单个代理的可用性 :param proxy_item: 代理信息 :param result_queue: 结果队列 """ result_payload = { "ok": False, "proxy": None, "reason": "invalid_proxy_item", } try: if 'ip' in proxy_item and 'port' in proxy_item: ip = proxy_item['ip'] port = proxy_item['port'] username = proxy_item.get('username', '') password = proxy_item.get('password', '') if username and password: proxy_url = f"http://{username}:{password}@{ip}:{port}" else: proxy_url = f"http://{ip}:{port}" import requests test_proxies = { 'http': proxy_url, 'https': proxy_url } result_payload["proxy"] = test_proxies test_targets = [ ("http://www.baidu.com", 8), ("https://m.baidu.com", 8), ("http://www.qq.com", 8), ("http://www.360.cn", 8), ] last_reason = "unknown" session = requests.Session() session.headers.update( { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/123.0.0.0 Safari/537.36" ) } ) for target_url, timeout_seconds in test_targets: try: test_response = session.get( target_url, proxies=test_proxies, timeout=timeout_seconds, allow_redirects=True, verify=False if target_url.startswith("https://") else True, ) if test_response.status_code in (200, 204, 301, 302, 403): result_payload["ok"] = True result_payload["reason"] = f"ok:{target_url}:{test_response.status_code}" break last_reason = f"http_status_{test_response.status_code}@{target_url}" except Exception as target_error: error_text = str(target_error).strip() or target_error.__class__.__name__ last_reason = f"{target_error.__class__.__name__}@{target_url}:{error_text[:160]}" session.close() if not result_payload["ok"]: result_payload["reason"] = last_reason except Exception as e: result_payload["reason"] = f"{e.__class__.__name__}:{str(e)[:160]}" finally: result_queue.put(result_payload) result_queue.put(None) def _extract_proxy_items(self, proxy_data): if isinstance(proxy_data, dict): if 'list' in proxy_data and isinstance(proxy_data['list'], list): return proxy_data['list'] if 'ip' in proxy_data and 'port' in proxy_data: return [proxy_data] if isinstance(proxy_data, list): return proxy_data return [] def _proxy_source_tag_from_url(self, raw_url): normalized_url = str(raw_url or "").strip() if not normalized_url: return "unknown" try: parsed = urllib.parse.urlsplit(normalized_url) query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) group_value = str((query.get("group") or [""])[0] or "").strip() if group_value: return group_value.upper() if parsed.netloc: return parsed.netloc except Exception: pass return normalized_url def _interleave_proxy_entries(self, proxy_entries): if not proxy_entries: return [] grouped_entries = collections.OrderedDict() for proxy_entry in proxy_entries: source_tag = str((proxy_entry or {}).get("source_tag") or "unknown").strip() or "unknown" grouped_entries.setdefault(source_tag, collections.deque()).append(proxy_entry) merged_entries = [] while grouped_entries: exhausted_tags = [] for source_tag, items in grouped_entries.items(): if not items: exhausted_tags.append(source_tag) continue merged_entries.append(items.popleft()) if not items: exhausted_tags.append(source_tag) for source_tag in exhausted_tags: grouped_entries.pop(source_tag, None) return merged_entries def _merge_proxy_entries(self, new_entries, existing_entries): merged = [] seen = set() for proxy_entry in list(new_entries or []) + list(existing_entries or []): if isinstance(proxy_entry, dict) and proxy_entry.get("proxy"): signature = self._proxy_key(proxy_entry.get("proxy")) else: signature = self._proxy_key(proxy_entry) if not signature or signature in seen: continue seen.add(signature) merged.append(proxy_entry) if merged and all(isinstance(item, dict) and item.get("proxy") for item in merged): return self._interleave_proxy_entries(merged) return merged def _proxy_signature_from_item(self, proxy_item): if not isinstance(proxy_item, dict): return "" ip = str(proxy_item.get("ip", "") or "").strip() port = str(proxy_item.get("port", "") or "").strip() username = str(proxy_item.get("username", "") or "").strip() password = str(proxy_item.get("password", "") or "").strip() if not ip or not port: return "" return json.dumps( { "ip": ip, "port": port, "username": username, "password": password, }, sort_keys=True, ensure_ascii=False, ) def _proxy_item_expire_timestamp(self, proxy_item): if not isinstance(proxy_item, dict): return 0.0 expire_at_ms = proxy_item.get("expire_at_ms") expire_at = proxy_item.get("expire_at") try: if expire_at_ms not in (None, "", 0, "0"): return float(expire_at_ms) / 1000.0 if expire_at not in (None, "", 0, "0"): return float(expire_at) except Exception: return 0.0 return 0.0 def _proxy_entry_expire_timestamp(self, proxy_entry): if isinstance(proxy_entry, dict): expire_at_ts = proxy_entry.get("expire_at_ts") try: if expire_at_ts not in (None, "", 0, "0"): return float(expire_at_ts) except Exception: return 0.0 raw_item = proxy_entry.get("raw_item") if isinstance(raw_item, dict): return float(self._proxy_item_expire_timestamp(raw_item) or 0.0) return float(self._proxy_item_expire_timestamp(proxy_entry) or 0.0) def _proxy_remaining_ttl_seconds(self, proxy_entry, *, now_ts=None): expire_at_ts = float(self._proxy_entry_expire_timestamp(proxy_entry) or 0.0) if expire_at_ts <= 0: return float("inf") observed_now_ts = float(now_ts or time.time()) return max(0.0, expire_at_ts - observed_now_ts) def _proxy_min_reuse_ttl_seconds(self, step_name=""): normalized_step_name = str(step_name or "").strip() override_map = { "注册状态检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_REGISTER", "6") or 6), "百度site检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_BAIDU", "8") or 8), "360检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_360", "8") or 8), "站长之家检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_CHINAZ", "10") or 10), "爱站网检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_AIZHAN", "10") or 10), "聚查检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUCHA", "12") or 12), "桔子检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUZISEO", "12") or 12), } if normalized_step_name in override_map: return max(0.0, float(override_map.get(normalized_step_name, 0.0) or 0.0)) configured_default = float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_SECONDS", "8") or 8) return max(0.0, configured_default) def _proxy_entry_can_be_reused(self, proxy_entry, step_name="", *, now_ts=None): remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(proxy_entry, now_ts=now_ts) if remaining_ttl_seconds == float("inf"): return True return remaining_ttl_seconds >= self._proxy_min_reuse_ttl_seconds(step_name) def _store_proxy_active_lease(self, proxy_entry, *, step_name="", already_locked=False): if not isinstance(proxy_entry, dict): return proxy = proxy_entry.get("proxy") proxy_key = self._proxy_key(proxy) if not proxy or not proxy_key: return def _store(): active_leases = getattr(self, "proxy_active_leases", None) if active_leases is None: self.proxy_active_leases = {} active_leases = self.proxy_active_leases lease_entry = dict(proxy_entry) lease_entry["leased_at"] = time.time() lease_entry["leased_step_name"] = str(step_name or proxy_entry.get("leased_step_name") or "").strip() active_leases[proxy_key] = lease_entry if already_locked: _store() return with self.proxy_pool_lock: _store() def _pop_proxy_active_lease(self, proxy, *, already_locked=False): proxy_key = self._proxy_key(proxy) if not proxy_key: return None def _pop(): active_leases = getattr(self, "proxy_active_leases", None) if not isinstance(active_leases, dict): return None return active_leases.pop(proxy_key, None) if already_locked: return _pop() with self.proxy_pool_lock: return _pop() def release_proxy(self, proxy, *, step_name="", discard=False, reason=""): if not proxy: return False with self.proxy_pool_lock: lease_entry = self._pop_proxy_active_lease(proxy, already_locked=True) if not isinstance(lease_entry, dict): return False remaining_count = len(self.proxy_pool) normalized_step_name = str(step_name or lease_entry.get("leased_step_name") or "").strip() source_tag = str(lease_entry.get("source_tag") or "").strip() reusable = ( not discard and not self._is_proxy_quarantined(proxy) and not self._is_proxy_source_quarantined(source_tag) and self._proxy_entry_can_be_reused(lease_entry, normalized_step_name) ) if reusable: lease_entry.pop("leased_at", None) lease_entry.pop("leased_step_name", None) with self.proxy_pool_lock: self.proxy_pool.append(lease_entry) remaining_count = len(self.proxy_pool) self._schedule_proxy_refresh_if_needed(remaining_count) if not reusable: remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(lease_entry) if remaining_ttl_seconds != float("inf"): logger.debug( f"代理归还时已丢弃: step={normalized_step_name or '-'} " f"ttl={int(remaining_ttl_seconds)}s reason={reason or ('discard' if discard else 'ttl_low')}" ) return reusable def _is_proxy_item_expired(self, proxy_item, *, grace_seconds=0.0): if not isinstance(proxy_item, dict): return True now_ts = time.time() expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0) if expire_at_ts <= 0: return False return expire_at_ts <= (now_ts - max(0.0, float(grace_seconds or 0.0))) def _summarize_proxy_expire_clock_skew(self, proxy_items, *, now_ts=None): if not isinstance(proxy_items, list) or not proxy_items: return None observed_now_ts = float(now_ts or time.time()) expire_timestamps = [] for proxy_item in proxy_items: expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0) if expire_at_ts > 0: expire_timestamps.append(expire_at_ts) if not expire_timestamps: return None latest_expire_at = max(expire_timestamps) earliest_expire_at = min(expire_timestamps) expired_timed_count = sum(1 for expire_at_ts in expire_timestamps if expire_at_ts <= observed_now_ts) # 所有带过期时间的代理都比本机时间早很多时，优先提示“时钟异常/源数据陈旧”， # 避免把问题误判成单纯的代理质量差。 if latest_expire_at > observed_now_ts - 900: return None if expired_timed_count < max(12, int(len(expire_timestamps) * 0.8)): return None return { "observed_now_ts": observed_now_ts, "observed_now_iso": datetime.fromtimestamp(observed_now_ts).strftime("%Y-%m-%d %H:%M:%S"), "timed_proxy_count": len(expire_timestamps), "expired_timed_count": expired_timed_count, "latest_expire_at": latest_expire_at, "latest_expire_at_iso": datetime.fromtimestamp(latest_expire_at).strftime("%Y-%m-%d %H:%M:%S"), "earliest_expire_at": earliest_expire_at, "earliest_expire_at_iso": datetime.fromtimestamp(earliest_expire_at).strftime("%Y-%m-%d %H:%M:%S"), "clock_ahead_seconds": max(0.0, observed_now_ts - latest_expire_at), } def _proxy_recently_expired_reuse_grace_seconds(self): configured_grace = float(os.getenv("DOMAINCHECK_PROXY_STALE_REUSE_GRACE_SECONDS", "0") or 0.0) if configured_grace > 0: return max(0.0, configured_grace) if not self.detecting: return 0.0 demand_threads = self._proxy_demand_threads() if demand_threads >= 1600: return 180.0 if demand_threads >= 1000: return 120.0 return 90.0 def _proxy_recently_expired_backfill_floor(self): configured_floor = int(os.getenv("DOMAINCHECK_PROXY_STALE_BACKFILL_FLOOR", "0") or 0) if configured_floor > 0: return max(8, configured_floor) demand_threads = self._proxy_demand_threads() return max(32, min(160, max(32, demand_threads // 12))) def _backfill_recently_expired_proxy_entries(self, current_entries, stale_entries): merged_entries = list(current_entries or []) normalized_stale_entries = list(stale_entries or []) if not self.detecting or not normalized_stale_entries: return merged_entries, 0, 0 floor = self._proxy_recently_expired_backfill_floor() if len(merged_entries) >= floor: return merged_entries, 0, floor before_count = len(merged_entries) needed = max(0, floor - before_count) if needed <= 0: return merged_entries, 0, floor merged_entries = self._merge_proxy_entries(normalized_stale_entries[:needed], merged_entries) return merged_entries, max(0, len(merged_entries) - before_count), floor def _set_proxy_request_count(self, raw_url, count): normalized_url = str(raw_url or "").strip() if not normalized_url: return "" try: parsed = urllib.parse.urlsplit(normalized_url) query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) query["count"] = [str(max(1, int(count or 0)))] return urllib.parse.urlunsplit( ( parsed.scheme, parsed.netloc, parsed.path, urllib.parse.urlencode(query, doseq=True), parsed.fragment, ) ) except Exception: return normalized_url def _prepare_proxy_fetch_plan(self, proxy_api_urls, current_pool_size=0): source_urls = [str(url or "").strip() for url in proxy_api_urls if str(url or "").strip()] if not source_urls: return [], { "target_total": 0, "batch_size": 0, "rounds": 0, "source_count": 0, } demand_threads = self._proxy_demand_threads() process_count = self._proxy_runtime_process_count() source_count = len(source_urls) provider_batch_cap = max(20, int(os.getenv("DOMAINCHECK_PROXY_PROVIDER_BATCH_CAP", "140") or 140)) provider_batch_floor = max( 20, min( provider_batch_cap, int(round(provider_batch_cap / self._proxy_multi_process_divisor())), ), ) current_pool_size = max(0, int(current_pool_size or 0)) refresh_threshold = self._proxy_refresh_threshold() shortage = max(0, refresh_threshold - current_pool_size) headroom = max(source_count * 10, int(refresh_threshold * 0.15)) active_round_cap = max(1, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_ACTIVE", "2") or 2)) idle_round_cap = max(2, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_IDLE", "4") or 4)) round_cap = active_round_cap if self.detecting else idle_round_cap target_total = max( source_count * provider_batch_floor, shortage + headroom, ) per_round_capacity = max(1, source_count * provider_batch_cap) rounds = max(1, min(round_cap, (target_total + per_round_capacity - 1) // per_round_capacity)) batch_size = max( 20, min( provider_batch_cap, (target_total + (source_count * rounds) - 1) // (source_count * rounds), ), ) fetch_plan = [] for round_index in range(rounds): for source_url in source_urls: fetch_plan.append( { "source_url": source_url, "request_url": self._set_proxy_request_count(source_url, batch_size), "batch_size": batch_size, "round": round_index + 1, } ) return fetch_plan, { "target_total": target_total, "batch_size": batch_size, "rounds": rounds, "source_count": source_count, "demand_threads": demand_threads, "process_count": process_count, "current_pool_size": current_pool_size, "refresh_threshold": refresh_threshold, "shortage": shortage, "provider_batch_floor": provider_batch_floor, } def _proxy_source_fetch_timeout_seconds(self): default_timeout = "4.0" if self.detecting else "6.0" return max( 1.5, float(os.getenv("DOMAINCHECK_PROXY_SOURCE_FETCH_TIMEOUT", default_timeout) or default_timeout), ) def _proxy_key(self, proxy): if isinstance(proxy, dict): return json.dumps(proxy, sort_keys=True, ensure_ascii=False) return str(proxy) def _is_proxy_quarantined(self, proxy): proxy_key = self._proxy_key(proxy) now_ts = time.time() with self.proxy_failure_lock: retry_after = float(self.proxy_quarantine_until.get(proxy_key, 0) or 0) if retry_after <= now_ts: if proxy_key in self.proxy_quarantine_until: self.proxy_quarantine_until.pop(proxy_key, None) self.proxy_failure_counts.pop(proxy_key, None) return False return True def _mark_proxy_failure(self, proxy): proxy_key = self._proxy_key(proxy) with self.proxy_failure_lock: failure_count = int(self.proxy_failure_counts.get(proxy_key, 0) or 0) + 1 self.proxy_failure_counts[proxy_key] = failure_count cooldown_seconds = min(180, 20 * failure_count) retry_after = time.time() + cooldown_seconds self.proxy_quarantine_until[proxy_key] = retry_after logger.info(f"代理临时隔离 {cooldown_seconds}s: {proxy}") return failure_count, cooldown_seconds def _clear_proxy_failure(self, proxy): proxy_key = self._proxy_key(proxy) with self.proxy_failure_lock: self.proxy_failure_counts.pop(proxy_key, None) self.proxy_quarantine_until.pop(proxy_key, None) def _proxy_source_tag_for_proxy(self, proxy): proxy_key = self._proxy_key(proxy) with self.proxy_failure_lock: source_map = getattr(self, "proxy_source_by_key", None) or {} return str(source_map.get(proxy_key, "") or "").strip() def _is_proxy_source_quarantined(self, source_tag): normalized_source_tag = str(source_tag or "").strip() if not normalized_source_tag: return False now_ts = time.time() with self.proxy_failure_lock: quarantine_map = getattr(self, "proxy_source_quarantine_until", None) or {} retry_after = float(quarantine_map.get(normalized_source_tag, 0) or 0) if retry_after <= now_ts: if normalized_source_tag in quarantine_map: quarantine_map.pop(normalized_source_tag, None) (getattr(self, "proxy_source_failure_counts", None) or {}).pop(normalized_source_tag, None) return False return True def _is_proxy_auth_issue(self, message): lowered = str(message or "").lower() return any( keyword in lowered for keyword in ( "407 proxy authentication required", "proxy authentication required", "tunnel connection failed: 407", "status code 407", ) ) def _remove_proxy_source_from_pool(self, source_tag): normalized_source_tag = str(source_tag or "").strip() if not normalized_source_tag: return 0 removed = 0 with self.proxy_pool_lock: filtered_pool = [] for proxy_item in list(getattr(self, "proxy_pool", []) or []): item_source_tag = "" if isinstance(proxy_item, dict): item_source_tag = str(proxy_item.get("source_tag") or "").strip() if item_source_tag and item_source_tag == normalized_source_tag: removed += 1 continue filtered_pool.append(proxy_item) if removed: self.proxy_pool = filtered_pool return removed def _mark_proxy_source_failure(self, proxy, reason=""): source_tag = self._proxy_source_tag_for_proxy(proxy) if not source_tag: return "", 0, 0, 0 with self.proxy_failure_lock: failure_counts = getattr(self, "proxy_source_failure_counts", None) if failure_counts is None: self.proxy_source_failure_counts = {} failure_counts = self.proxy_source_failure_counts quarantine_map = getattr(self, "proxy_source_quarantine_until", None) if quarantine_map is None: self.proxy_source_quarantine_until = {} quarantine_map = self.proxy_source_quarantine_until failure_count = int(failure_counts.get(source_tag, 0) or 0) + 1 failure_counts[source_tag] = failure_count if self._is_proxy_auth_issue(reason): cooldown_seconds = min(600, max(90, 90 * failure_count)) else: cooldown_seconds = min(300, max(30, 30 * failure_count)) quarantine_map[source_tag] = time.time() + cooldown_seconds removed = self._remove_proxy_source_from_pool(source_tag) logger.warning( f"代理来源已临时隔离 {cooldown_seconds}s: source={source_tag}, failures={failure_count}, removed={removed}, reason={reason}" ) return source_tag, failure_count, cooldown_seconds, removed def _clear_proxy_source_failure(self, proxy): source_tag = self._proxy_source_tag_for_proxy(proxy) if not source_tag: return with self.proxy_failure_lock: (getattr(self, "proxy_source_failure_counts", None) or {}).pop(source_tag, None) (getattr(self, "proxy_source_quarantine_until", None) or {}).pop(source_tag, None) def _schedule_proxy_refresh_if_needed(self, available_count=None): if available_count is None: with self.proxy_pool_lock: current_count = len(self.proxy_pool) else: # 调用方可能已经持有 proxy_pool_lock；避免对同一把非重入锁再次加锁。 current_count = int(available_count or 0) threshold = self._proxy_refresh_threshold() if float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time(): return if current_count < threshold and not self.proxy_refresh_lock.locked(): threading.Thread(target=self.refresh_proxy_pool, daemon=True).start() def _has_proxy_sources(self): proxy_urls = self.proxy_config.get('proxy_urls') or [] if not proxy_urls and self.proxy_config.get('proxy_url'): proxy_urls = [self.proxy_config.get('proxy_url', '')] return bool([url for url in proxy_urls if str(url or '').strip()]) def trigger_proxy_refresh(self, reason="", reset_cooldown=False, debounce_seconds=None): """ 代理配置一旦下发，就允许在非检测态主动补一次刷新，避免页面长期停在“未刷新”。 """ if not self.proxy_config.get('proxy_enable', False): return False if not self._has_proxy_sources(): return False normalized_reason = str(reason or "manual").strip() or "manual" if debounce_seconds is None and normalized_reason.startswith("config_update:"): debounce_seconds = self._config_update_proxy_refresh_debounce_seconds() debounce_seconds = max(0.0, float(debounce_seconds or 0.0)) refresh_reason_timestamps = getattr(self, "_last_proxy_refresh_reason_at", None) if not isinstance(refresh_reason_timestamps, dict): refresh_reason_timestamps = {} self._last_proxy_refresh_reason_at = refresh_reason_timestamps trigger_key = "config_update" if normalized_reason.startswith("config_update:") else normalized_reason now_ts = time.time() last_trigger_at = float(refresh_reason_timestamps.get(trigger_key, 0.0) or 0.0) if debounce_seconds > 0 and (now_ts - last_trigger_at) < debounce_seconds: logger.debug( f"代理池刷新触发去抖，跳过主动触发: {normalized_reason}, cooldown={debounce_seconds:.1f}s" ) return False if reset_cooldown: self.proxy_next_refresh_time = 0.0 elif float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time(): logger.debug(f"代理池仍在冷却窗口，跳过主动触发: {normalized_reason}") return False if self.proxy_refresh_lock.locked(): logger.debug(f"代理池刷新已在进行中，跳过主动触发: {normalized_reason}") return False refresh_reason_timestamps[trigger_key] = now_ts logger.info(f"主动触发代理池刷新: {normalized_reason}") threading.Thread(target=self.refresh_proxy_pool, daemon=True).start() return True def _proxy_refresh_threshold(self): """ 代理池的补货阈值只用于触发后台刷新，不应反向限制并发。 """ demand_threads = self._proxy_demand_threads() base_threshold = max(840, demand_threads * 3) process_count = self._proxy_runtime_process_count() if process_count <= 1: return base_threshold scaled_threshold = int(round(base_threshold / self._proxy_multi_process_divisor())) process_floor = max(120, min(480, max(120, demand_threads // 4))) return max(process_floor, scaled_threshold) def _proxy_demand_threads(self): configured_threads = max(1, int(getattr(self, "thread_count", 0) or 0)) live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) if self.detecting: warm_floor = min(configured_threads, max(400, configured_threads // 2)) demand_threads = max(live_active_threads, warm_floor) else: demand_threads = min(configured_threads, 64) return max(1, demand_threads) def _should_emit_thread_progress(self, current_active, max_threads, last_emit_count, last_emit_at): """ 线程创建/回收阶段的活跃数上报做节流，避免每起一个线程就同步 一次远端日志和运行态，反向把并发拉低。 """ now = time.time() step = max(1, min(10, max_threads // 10)) if current_active == 1: return True, now if current_active >= max_threads: return True, now if current_active == 0 and last_emit_count != 0: return True, now if current_active - last_emit_count >= step: return True, now if current_active > 0 and now - last_emit_at >= 5.0: return True, now return False, now def _wait_for_proxy_refresh_settle(self, timeout_seconds=2.0): deadline = time.time() + max(0.0, float(timeout_seconds or 0.0)) while self.proxy_refresh_lock.locked() and time.time() < deadline: with self.proxy_pool_lock: if self.proxy_pool: return True time.sleep(0.05) with self.proxy_pool_lock: return bool(self.proxy_pool) def _prepare_proxy_pool_for_detection_start(self): """ 检测会话启动时，优先复用当前可用代理池/共享快照。 只有在本地和共享快照都为空时，才做同步刷新。 否则 large controller 会在每轮 auto-resume 前都卡一次 refresh_proxy_pool， 线程池还没起就先停在 refreshing_proxy。 """ with self.proxy_pool_lock: current_pool_size = len(self.proxy_pool) if current_pool_size > 0: self._schedule_proxy_refresh_if_needed(current_pool_size) return True, current_pool_size, "local_pool_ready" if self._proxy_should_coordinate_shared_refresh(): shared_payload = self._load_shared_proxy_snapshot_payload() if shared_payload and self._apply_shared_proxy_snapshot( shared_payload, status_prefix="检测启动复用共享代理快照", ): with self.proxy_pool_lock: shared_pool_size = len(self.proxy_pool) self._schedule_proxy_refresh_if_needed(shared_pool_size) return True, shared_pool_size, "shared_snapshot_ready" return False, 0, "proxy_pool_empty" def refresh_proxy_pool(self): """ 刷新代理池 """ if not self.proxy_refresh_lock.acquire(blocking=False): logger.debug("代理池刷新已在进行中，跳过本次重复刷新") return shared_refresh_owner = "" if not self.proxy_config.get('proxy_enable', False): self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_status = "代理未启用" self.proxy_last_refresh_source_count = 0 self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = 0 self.proxy_last_source_stats = [] self.proxy_next_refresh_time = 0.0 self._update_runtime_state( "idle" if not self.detecting else "running", "代理未启用，使用直接连接", ) self._sync_worker_log_event("代理未启用，使用直接连接", mode='key') self.proxy_refresh_lock.release() return try: now_ts = time.time() retry_delay_seconds = 0.15 if self.detecting else min(self.proxy_refresh_cooldown_seconds, 10) holdoff_seconds = self._proxy_refresh_holdoff_seconds() refresh_threshold = self._proxy_refresh_threshold() with self.proxy_pool_lock: has_cached_proxies = bool(self.proxy_pool) cached_proxy_pool = list(self.proxy_pool) if self._proxy_should_coordinate_shared_refresh(): shared_payload = self._load_shared_proxy_snapshot_payload() if shared_payload: shared_entries = shared_payload.get("proxy_pool") or [] if self._shared_proxy_snapshot_should_replace_local_pool( len(shared_entries), len(cached_proxy_pool), refresh_threshold, ): if self._apply_shared_proxy_snapshot(shared_payload): self.proxy_next_refresh_time = time.time() + holdoff_seconds return if not has_cached_proxies and self.proxy_next_refresh_time and now_ts < self.proxy_next_refresh_time: wait_seconds = int(max(1, self.proxy_next_refresh_time - now_ts)) self.proxy_last_refresh_status = f"冷却中，{wait_seconds} 秒后再试" logger.info(f"代理池刷新冷却中，{wait_seconds} 秒后再试") self._update_runtime_state( "refreshing_proxy" if self.detecting else "idle", f"代理池刷新冷却中，{wait_seconds} 秒后再试", ) self._sync_worker_log_event(f"代理池刷新冷却中，{wait_seconds} 秒后再试", mode='full') return if self._proxy_should_coordinate_shared_refresh(): shared_refresh_owner = self._acquire_shared_proxy_refresh_lock() if not shared_refresh_owner: if self._wait_for_shared_proxy_snapshot(): self.proxy_next_refresh_time = time.time() + holdoff_seconds return if cached_proxy_pool: with self.proxy_pool_lock: self.proxy_pool = cached_proxy_pool self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = len(self.proxy_pool) self.proxy_last_refresh_status = f"共享刷新进行中，继续沿用缓存 {len(cached_proxy_pool)} 个" else: self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = 0 self.proxy_last_refresh_status = "共享代理刷新进行中，等待补货" self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds self._update_runtime_state( "refreshing_proxy" if self.detecting else "idle", self.proxy_last_refresh_status, ) self._sync_worker_log_event(self.proxy_last_refresh_status, mode='key') return proxy_api_urls = self.proxy_config.get('proxy_urls') or [] if not proxy_api_urls and self.proxy_config.get('proxy_url'): proxy_api_urls = [self.proxy_config.get('proxy_url', '')] proxy_api_urls = [url for url in proxy_api_urls if url] proxy_fetch_plan, fetch_plan_meta = self._prepare_proxy_fetch_plan( proxy_api_urls, current_pool_size=len(cached_proxy_pool), ) source_count = int(fetch_plan_meta.get("source_count", 0) or 0) batch_size = int(fetch_plan_meta.get("batch_size", 0) or 0) target_total = int(fetch_plan_meta.get("target_total", 0) or 0) rounds = int(fetch_plan_meta.get("rounds", 0) or 0) refresh_threshold = int(fetch_plan_meta.get("refresh_threshold", 0) or refresh_threshold) self.proxy_last_refresh_source_count = source_count self.proxy_last_source_stats = [] if proxy_fetch_plan: import requests proxy_list = [] cooldown_detected = False aggressive_proxy_rotation = str(os.getenv("DOMAINCHECK_PROXY_FAST_ROTATION", "1") or "1").strip().lower() not in { "", "0", "false", "no", "off", } def fetch_proxy_source(plan_item): source_url = str(plan_item.get("source_url") or "").strip() requested_batch_size = max(1, int(plan_item.get("batch_size", 0) or batch_size or 1)) round_index = int(plan_item.get("round", 1) or 1) source_stat = { "url": source_url, "source_url": source_url, "status": "unknown", "http_status": 0, "raw_items": 0, "error": "", "requested_batch_size": requested_batch_size, "effective_batch_size": requested_batch_size, "round": round_index, } current_items = [] batch_attempts = [] if self.detecting and aggressive_proxy_rotation: dynamic_fallbacks = [ requested_batch_size, max(80, requested_batch_size // 2), 80, 50, 20, 10, ] else: dynamic_fallbacks = [ requested_batch_size, max(500, requested_batch_size // 2), 200, 100, 50, 20, 10, ] for candidate in dynamic_fallbacks: candidate = max(1, int(candidate or 0)) if candidate not in batch_attempts: batch_attempts.append(candidate) for candidate_batch_size in batch_attempts: request_url = self._set_proxy_request_count(source_url, candidate_batch_size) source_stat["effective_batch_size"] = candidate_batch_size try: response = requests.get( request_url, timeout=self._proxy_source_fetch_timeout_seconds(), ) source_stat["http_status"] = int(response.status_code or 0) if response.status_code != 200: source_stat["status"] = "http_error" source_stat["error"] = f"HTTP {response.status_code}" logger.warning(f"代理池链接响应异常: {request_url}, 状态码: {response.status_code}") continue proxy_data = response.json() current_items = self._extract_proxy_items(proxy_data) source_tag = self._proxy_source_tag_from_url(source_url) annotated_items = [] for proxy_item in current_items: if isinstance(proxy_item, dict): item_copy = dict(proxy_item) item_copy.setdefault("source_url", source_url) item_copy.setdefault("source_tag", source_tag) annotated_items.append(item_copy) current_items = annotated_items source_stat["raw_items"] = len(current_items) if current_items: source_stat["status"] = "ok" logger.info( f"代理池链接拉取成功: {request_url}, 原始代理数: {len(current_items)}, 轮次: {round_index}" ) break error_message = "" if isinstance(proxy_data, dict): error_message = str(proxy_data.get("error") or "").strip() source_stat["status"] = "cooldown" if "cooldown" in error_message.lower() else "empty" source_stat["error"] = error_message or "empty" logger.info( f"代理池链接空载，降批重试: {request_url}, 轮次: {round_index}, 提示: {source_stat['error']}" ) if source_stat["status"] == "cooldown": break except Exception as api_error: source_stat["status"] = "request_error" source_stat["error"] = str(api_error) logger.warning(f"代理池链接拉取失败: {request_url}, 错误: {api_error}") source_stat["url"] = self._set_proxy_request_count(source_url, source_stat["effective_batch_size"]) return source_stat, current_items round_plan_map = {} for plan_item in proxy_fetch_plan: round_key = max(1, int(plan_item.get("round", 1) or 1)) round_plan_map.setdefault(round_key, []).append(plan_item) round_keys = sorted(round_plan_map.keys()) per_round_workers = max(1, min(source_count or len(proxy_api_urls) or 1, 12)) cooled_source_urls = set() for round_index in round_keys: round_plan_items = [ item for item in (round_plan_map.get(round_index) or []) if str(item.get("source_url") or "").strip() not in cooled_source_urls ] if not round_plan_items: continue with ThreadPoolExecutor(max_workers=min(per_round_workers, len(round_plan_items))) as executor: futures = [executor.submit(fetch_proxy_source, plan_item) for plan_item in round_plan_items] for future in as_completed(futures): source_stat, current_items = future.result() if "cooldown" in str(source_stat.get("error") or "").lower() or source_stat.get("status") == "cooldown": cooldown_detected = True cooled_source_urls.add(str(source_stat.get("source_url") or "").strip()) self.proxy_last_source_stats.append(source_stat) proxy_list.extend(current_items) if len(proxy_list) >= target_total: break if self.detecting and aggressive_proxy_rotation and round_index < len(round_keys): time.sleep(0.15 if cooldown_detected else 0.05) if proxy_list: self.proxy_last_refresh_total_items = len(proxy_list) self.proxy_last_validated_count = 0 new_proxies = [] fallback_stale_proxies = [] seen_proxy_strings = set() expired_count = 0 invalid_count = 0 stale_reuse_grace_seconds = self._proxy_recently_expired_reuse_grace_seconds() for proxy_item in proxy_list: signature = self._proxy_signature_from_item(proxy_item) if not signature: invalid_count += 1 continue if signature in seen_proxy_strings: continue seen_proxy_strings.add(signature) ip = str(proxy_item.get("ip", "") or "").strip() port = str(proxy_item.get("port", "") or "").strip() username = str(proxy_item.get("username", "") or "").strip() password = str(proxy_item.get("password", "") or "").strip() proxy_url = f"http://{username}:{password}@{ip}:{port}" if username and password else f"http://{ip}:{port}" source_tag = self._proxy_source_tag_from_url(proxy_item.get("source_url") or proxy_item.get("source_tag") or "") if self._is_proxy_source_quarantined(source_tag): continue proxy_entry = { 'proxy': { 'http': proxy_url, 'https': proxy_url, }, 'usage_count': 0, 'source_tag': source_tag, 'expire_at_ts': float(self._proxy_item_expire_timestamp(proxy_item) or 0.0), 'raw_item': dict(proxy_item), } if self._is_proxy_item_expired(proxy_item): expired_count += 1 if ( stale_reuse_grace_seconds > 0 and not self._is_proxy_item_expired( proxy_item, grace_seconds=stale_reuse_grace_seconds, ) ): fallback_stale_proxies.append(proxy_entry) continue new_proxies.append(proxy_entry) clock_skew_hint = self._summarize_proxy_expire_clock_skew(proxy_list) if clock_skew_hint: now_ts = time.time() last_warning_at = float(getattr(self, "_last_proxy_clock_skew_warning_at", 0.0) or 0.0) if now_ts - last_warning_at >= 30.0: self._last_proxy_clock_skew_warning_at = now_ts warning_message = ( "疑似系统时钟异常或代理源返回陈旧数据: " f"本机时间 {clock_skew_hint['observed_now_iso']}，" f"代理最晚过期 {clock_skew_hint['latest_expire_at_iso']}，" f"偏移约 {int(clock_skew_hint['clock_ahead_seconds'])} 秒" ) logger.warning( warning_message + ( f"，带过期时间代理 {clock_skew_hint['timed_proxy_count']} 个，" f"其中已过期 {clock_skew_hint['expired_timed_count']} 个" ) ) self._sync_worker_log_event( warning_message, level='warning', payload=clock_skew_hint, mode='key', ) fallback_stale_count = 0 stale_backfill_count = 0 stale_backfill_floor = 0 if not new_proxies and fallback_stale_proxies and not cached_proxy_pool: new_proxies = fallback_stale_proxies fallback_stale_count = len(new_proxies) merged_proxy_pool = [] if new_proxies: merged_proxy_pool = self._merge_proxy_entries(new_proxies, cached_proxy_pool) elif cached_proxy_pool: merged_proxy_pool = list(cached_proxy_pool) if fallback_stale_proxies and stale_reuse_grace_seconds > 0: ( merged_proxy_pool, stale_backfill_count, stale_backfill_floor, ) = self._backfill_recently_expired_proxy_entries( merged_proxy_pool, fallback_stale_proxies, ) with self.proxy_pool_lock: self.proxy_pool = merged_proxy_pool source_map = {} for proxy_entry in merged_proxy_pool: if isinstance(proxy_entry, dict) and proxy_entry.get("proxy"): source_map[self._proxy_key(proxy_entry.get("proxy"))] = str( proxy_entry.get("source_tag") or "" ).strip() with self.proxy_failure_lock: self.proxy_source_by_key = source_map self.proxy_last_refresh_time = datetime.now() self.proxy_last_available_count = len(self.proxy_pool) if new_proxies: self._publish_shared_proxy_snapshot( merged_proxy_pool, source_count=source_count, raw_items=self.proxy_last_refresh_total_items, ) if new_proxies: if fallback_stale_count > 0: self.proxy_last_refresh_status = ( f"疑似过期回退增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）" ) elif stale_backfill_count > 0: self.proxy_last_refresh_status = ( f"最近过期代理回补 {stale_backfill_count} 个，当前池 {len(self.proxy_pool)} 个" ) else: self.proxy_last_refresh_status = ( f"增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）" ) self.proxy_next_refresh_time = ( 0.0 if len(self.proxy_pool) >= refresh_threshold else (time.time() + holdoff_seconds) ) elif cached_proxy_pool: self.proxy_last_refresh_status = f"本轮未拿到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个" next_retry_delay = ( 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) ) next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay else: self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据" next_retry_delay = ( 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) ) next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay logger.info( f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，" f"原始 {self.proxy_last_refresh_total_items} 个，去过期 {expired_count} 个，非法 {invalid_count} 个，" f"目标总量 {target_total}，单批 {batch_size}，轮次 {rounds}" ) self._update_runtime_state( "running" if self.detecting else "idle", f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，原始 {self.proxy_last_refresh_total_items} 个，单批 {batch_size}，轮次 {rounds}", ) self._sync_worker_log_event( f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，原始 {self.proxy_last_refresh_total_items} 个，单批 {batch_size}，轮次 {rounds}", payload={ "available_proxy_count": len(self.proxy_pool), "source_count": source_count, "raw_items": self.proxy_last_refresh_total_items, "validated_count": 0, "expired_count": expired_count, "invalid_count": invalid_count, "fallback_stale_count": fallback_stale_count, "stale_backfill_count": stale_backfill_count, "stale_backfill_floor": stale_backfill_floor, "stale_reuse_grace_seconds": stale_reuse_grace_seconds, "target_total": target_total, "batch_size": batch_size, "rounds": rounds, }, mode='key', ) else: with self.proxy_pool_lock: self.proxy_pool = cached_proxy_pool self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = len(self.proxy_pool) if cached_proxy_pool: self.proxy_last_refresh_status = f"未取到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个" else: self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据" next_retry_delay = ( 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) ) next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay logger.warning("所有代理池链接均未返回可用代理数据") self._update_runtime_state( "refreshing_proxy" if self.detecting else "idle", self.proxy_last_refresh_status, ) self._sync_worker_log_event(self.proxy_last_refresh_status, level='warning', mode='key') else: with self.proxy_pool_lock: self.proxy_pool = cached_proxy_pool self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = len(self.proxy_pool) self.proxy_last_refresh_status = "未配置代理池链接" self.proxy_next_refresh_time = time.time() + ( (0.5 if (self.detecting and aggressive_proxy_rotation) else retry_delay_seconds) + holdoff_seconds ) self._update_runtime_state( "idle" if not self.detecting else "running", "未配置代理池链接", ) self._sync_worker_log_event("未配置代理池链接", level='warning', mode='key') except Exception as e: with self.proxy_pool_lock: cached_proxy_pool = list(self.proxy_pool) self.proxy_pool = cached_proxy_pool self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_total_items = 0 self.proxy_last_validated_count = 0 self.proxy_last_available_count = len(self.proxy_pool) if cached_proxy_pool: self.proxy_last_refresh_status = f"刷新失败，继续沿用缓存 {len(cached_proxy_pool)} 个: {e}" else: self.proxy_last_refresh_status = f"刷新失败: {e}" self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds logger.error(f"刷新代理池失败: {e}") self._update_runtime_state( "refreshing_proxy" if self.detecting else "failed", self.proxy_last_refresh_status, ) self._sync_worker_log_event(self.proxy_last_refresh_status, level='error', mode='key') finally: self._release_shared_proxy_refresh_lock(shared_refresh_owner) self.proxy_refresh_lock.release() def remove_proxy(self, proxy): """ 从代理池中移除失效的代理 :param proxy: 失效的代理 """ failure_count, cooldown_seconds = self._mark_proxy_failure(proxy) with self.proxy_pool_lock: remaining_count = len(self.proxy_pool) removed = 0 if self._pop_proxy_active_lease(proxy, already_locked=True): removed += 1 if self.proxy_pool and isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]: filtered_pool = [] for proxy_item in self.proxy_pool: if proxy_item.get('proxy') == proxy: removed += 1 continue filtered_pool.append(proxy_item) if removed: self.proxy_pool = filtered_pool remaining_count = len(self.proxy_pool) logger.info(f"代理失效已从当前池移除 {removed} 个: {proxy}") else: filtered_pool = [] for existing_proxy in self.proxy_pool: if existing_proxy == proxy: removed += 1 continue filtered_pool.append(existing_proxy) if removed: self.proxy_pool = filtered_pool remaining_count = len(self.proxy_pool) logger.info(f"代理失效已从当前池移除 {removed} 个: {proxy}") logger.info( f"代理失败计数已更新: 第 {failure_count} 次失败，隔离 {cooldown_seconds}s，当前池内代理 {remaining_count} 个" ) self._schedule_proxy_refresh_if_needed(remaining_count) def _proxy_pool_scan_batch_size(self): configured_batch_size = int(os.getenv("DOMAINCHECK_PROXY_POOL_SCAN_BATCH_SIZE", "0") or 0) if configured_batch_size > 0: return max(8, configured_batch_size) demand_threads = self._proxy_demand_threads() if demand_threads >= 1600: return 64 if demand_threads >= 800: return 48 return 32 def _pop_proxy_candidate_batch(self, batch_size): normalized_batch_size = max(1, int(batch_size or 1)) with self.proxy_pool_lock: if not self.proxy_pool: return [] actual_batch_size = min(len(self.proxy_pool), normalized_batch_size) candidates = [] for _ in range(actual_batch_size): candidates.append(self.proxy_pool.pop(0)) return candidates def _restore_proxy_candidate_batch(self, candidates, selected_entry=None, *, selected_step_name="", lease_selected=False): restored_count = 0 with self.proxy_pool_lock: if candidates: self.proxy_pool.extend(candidates) restored_count += len(candidates) if selected_entry is not None: if lease_selected: self._store_proxy_active_lease( selected_entry, step_name=selected_step_name, already_locked=True, ) else: self.proxy_pool.append(selected_entry) restored_count += 1 remaining_count = len(self.proxy_pool) return remaining_count, restored_count def get_proxies(self, excluded_proxy_keys=None, step_name="", lease_selected=False): """ 获取代理配置 """ excluded_proxy_keys = set(excluded_proxy_keys or []) if self.proxy_config.get('proxy_enable', False): with self.proxy_pool_lock: current_pool_size = len(self.proxy_pool) is_empty = current_pool_size == 0 is_insufficient = current_pool_size < self._proxy_refresh_threshold() if is_empty: if self._wait_for_shared_proxy_snapshot(timeout_seconds=min(0.25, self._shared_proxy_wait_timeout_seconds())): with self.proxy_pool_lock: current_pool_size = len(self.proxy_pool) is_empty = current_pool_size == 0 is_insufficient = current_pool_size < self._proxy_refresh_threshold() # 不要在检测线程里同步阻塞等待代理刷新，否则线程数上去了也会卡死在这里。 if is_empty: self._schedule_proxy_refresh_if_needed(current_pool_size) elif is_insufficient: logger.debug( f"代理池低水位，触发后台补货: 当前 {current_pool_size}，阈值 {self._proxy_refresh_threshold()}" ) self._schedule_proxy_refresh_if_needed(current_pool_size) if current_pool_size > 0: scan_batch_size = self._proxy_pool_scan_batch_size() scanned_count = 0 while scanned_count < current_pool_size: candidates = self._pop_proxy_candidate_batch( min(scan_batch_size, max(1, current_pool_size - scanned_count)) ) if not candidates: break scanned_count += len(candidates) deferred_candidates = [] selected_proxy = None selected_entry = None observed_now_ts = time.time() for index, candidate in enumerate(candidates): if isinstance(candidate, dict) and candidate.get('proxy'): proxy = candidate['proxy'] proxy_key = self._proxy_key(proxy) if self._is_proxy_quarantined(proxy): deferred_candidates.append(candidate) continue source_tag = str(candidate.get("source_tag") or "").strip() if self._is_proxy_source_quarantined(source_tag): deferred_candidates.append(candidate) continue if proxy_key in excluded_proxy_keys: deferred_candidates.append(candidate) continue if lease_selected and not self._proxy_entry_can_be_reused(candidate, step_name, now_ts=observed_now_ts): continue usage_count = int(candidate.get('usage_count', 0)) + 1 candidate['usage_count'] = 0 if usage_count >= self.proxy_max_reuse_count else usage_count selected_proxy = proxy selected_entry = candidate with self.proxy_failure_lock: source_map = getattr(self, "proxy_source_by_key", None) if source_map is None: self.proxy_source_by_key = {} source_map = self.proxy_source_by_key source_map[proxy_key] = source_tag deferred_candidates.extend(candidates[index + 1 :]) break proxy = candidate proxy_key = self._proxy_key(proxy) if self._is_proxy_quarantined(proxy): deferred_candidates.append(candidate) continue if proxy_key in excluded_proxy_keys: deferred_candidates.append(candidate) continue logger.info(f"从代理池选择代理: {proxy}") selected_proxy = proxy selected_entry = candidate if isinstance(candidate, dict) else { "proxy": proxy, "usage_count": 0, "source_tag": "", } deferred_candidates.extend(candidates[index + 1 :]) break if selected_proxy is None: deferred_candidates = candidates remaining_count, _ = self._restore_proxy_candidate_batch( deferred_candidates, selected_entry=selected_entry, selected_step_name=step_name, lease_selected=lease_selected, ) if selected_proxy is not None: self._schedule_proxy_refresh_if_needed(remaining_count) return selected_proxy if self.proxy_config.get('proxy_enable', False): now_ts = time.time() if now_ts - float(getattr(self, "_last_no_proxy_notice_at", 0.0) or 0.0) >= 5.0: self._last_no_proxy_notice_at = now_ts logger.warning("代理已启用，但当前无可用代理") fallback_detail = ( f"代理已启用，但当前无可用代理；已允许直连兜底，最近代理状态：{self.proxy_last_refresh_status}" if self.allow_direct_connection() else f"代理已启用，但当前无可用代理；且未允许直连，最近代理状态：{self.proxy_last_refresh_status}" ) self._update_runtime_state( "running" if self.detecting else "idle", fallback_detail, ) else: logger.info("未启用代理，使用直接连接") return None def allow_direct_connection(self): return bool(self.proxy_config.get('allow_direct', False)) @staticmethod def _normalize_flag_value(value, default=False): if value is None: return bool(default) if isinstance(value, bool): return value return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"} @staticmethod def _env_flag_enabled(env_name, default=False): value = os.getenv(env_name) return DetectWorker._normalize_flag_value(value, default=default) def _register_single_machine_mode_enabled(self): cached = getattr(self, "register_single_machine_mode_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled("DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE") def _register_single_machine_direct_streak_attempts(self): cached = getattr(self, "register_single_machine_direct_streak_attempts", None) if cached is not None: return max(0, int(cached or 0)) if not self._register_single_machine_mode_enabled(): return 0 return max(0, int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2)) def _single_machine_site_direct_fallback_enabled(self): cached = getattr(self, "single_machine_site_direct_fallback_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK", default=self._register_single_machine_mode_enabled(), ) def _single_machine_aizhan_direct_first_enabled(self): cached = getattr(self, "single_machine_aizhan_direct_first_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST", default=False, ) def _single_machine_baidu_direct_first_enabled(self): cached = getattr(self, "single_machine_baidu_direct_first_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST", default=False, ) def _single_machine_chinaz_direct_first_enabled(self): cached = getattr(self, "single_machine_chinaz_direct_first_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_SINGLE_MACHINE_CHINAZ_DIRECT_FIRST", default=False, ) def _aizhan_remote_disconnect_degrade_enabled(self): cached = getattr(self, "aizhan_remote_disconnect_degrade_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE", default=False, ) def _aizhan_external_fast_degrade_enabled(self): cached = getattr(self, "aizhan_external_fast_degrade_enabled", None) if cached is not None: return bool(cached) return self._env_flag_enabled( "DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE", default=False, ) def _should_force_direct_first_attempt(self, step_name, attempt_count): if int(attempt_count or 0) != 0: return False if ( step_name == "爱站网检测" and self._single_machine_aizhan_direct_first_enabled() and self._allow_direct_connection_for_step(step_name) ): return True if ( step_name == "百度site检测" and self._single_machine_baidu_direct_first_enabled() and self._allow_direct_connection_for_step(step_name) ): return True if ( step_name == "站长之家检测" and self._single_machine_chinaz_direct_first_enabled() and self._allow_direct_connection_for_step(step_name) ): return True return False def _should_degrade_remote_disconnect_immediately(self, step_name, reason): if step_name != "爱站网检测" or not self._aizhan_remote_disconnect_degrade_enabled(): return False lowered = str(reason or "").strip().lower() if not lowered: return False return ( "remotedisconnected" in lowered or "remote end closed connection without response" in lowered ) def _should_degrade_external_issue_immediately(self, step_name, reason): if step_name != "爱站网检测": return False if self._should_degrade_remote_disconnect_immediately(step_name, reason): return True if not self._aizhan_external_fast_degrade_enabled(): return False return self._is_external_dependency_issue(reason) def _allow_direct_connection_for_step(self, step_name): if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): return True if ( step_name in {"360检测", "百度site检测", "站长之家检测", "爱站网检测"} and self._single_machine_site_direct_fallback_enabled() ): return True return self.allow_direct_connection() def _proxy_status_indicates_shortage(self): status = str(getattr(self, "proxy_last_refresh_status", "") or "").strip() if not status: return False keywords = ( "冷却中", "暂时冷却", "无可用代理", "未取到可用代理", "继续沿用缓存 0", "稍后继续补货", ) return any(keyword in status for keyword in keywords) def _proxy_pool_available_count(self): try: with self.proxy_pool_lock: return len(list(getattr(self, "proxy_pool", []) or [])) except Exception: return len(list(getattr(self, "proxy_pool", []) or [])) def _should_hold_stale_detect_for_proxy_shortage( self, *, idle_seconds: float, low_activity_recoverable: bool, incoming_job_changed: bool, ): if incoming_job_changed or not low_activity_recoverable: return False, "" if not bool(self.proxy_config.get('proxy_enable', False)): return False, "" if not self._proxy_status_indicates_shortage(): return False, "" if self._proxy_pool_available_count() > 0: return False, "" grace_seconds = 120.0 if self.allow_direct_connection() else 180.0 if idle_seconds >= grace_seconds: return False, "" return True, ( f"代理池仍处于短缺/冷却窗口，当前空转 {int(idle_seconds)} 秒，" f"小于保护阈值 {int(grace_seconds)} 秒，最近代理状态：{self.proxy_last_refresh_status}" ) def _proxy_direct_fallback_grace_for_step(self, step_name): default_grace = max(0.0, float(getattr(self, "proxy_direct_fallback_grace_seconds", 0.0) or 0.0)) override_map = getattr(self, "proxy_direct_fallback_grace_overrides", None) or {} step_grace = max(0.0, float(override_map.get(step_name, default_grace) or 0.0)) if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): return 0.0 if not self._allow_direct_connection_for_step(step_name): return step_grace if ( step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"} and self._proxy_status_indicates_shortage() ): return min(step_grace, 0.05) return step_grace def _proxy_direct_retry_wait_for_step(self, step_name): default_wait = max(0.0, float(getattr(self, "proxy_direct_retry_wait_seconds", 0.0) or 0.0)) override_map = getattr(self, "proxy_direct_retry_wait_overrides", None) or {} step_wait = max(0.0, float(override_map.get(step_name, default_wait) or 0.0)) if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): return 0.0 if ( step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"} and self._proxy_status_indicates_shortage() ): return min(step_wait, 0.25) return step_wait def _get_proxy_for_step(self, domain_id, domain_name, step_name, excluded_proxy_keys=None): started_at = time.perf_counter() excluded_proxy_keys = set(excluded_proxy_keys or []) proxy_enabled = bool(self.proxy_config.get('proxy_enable', False)) direct_allowed = self._allow_direct_connection_for_step(step_name) force_direct_once = self._consume_step_force_direct_once(step_name) step_direct_grace_seconds = self._proxy_direct_fallback_grace_for_step(step_name) if proxy_enabled and direct_allowed and force_direct_once: self._emit_step_trace( domain_name, step_name, "proxy_direct_fallback", elapsed_ms=0, proxy_mode="direct", recent_proxy_status=self.proxy_last_refresh_status, retry_attempts=0, reason="external_retry_direct_fallback", ) return None direct_retry_wait_seconds = self._consume_step_proxy_retry_wait(step_name) wait_window_seconds = 0.0 if proxy_enabled and not direct_allowed: wait_window_seconds = self.proxy_step_wait_timeout_seconds elif proxy_enabled and direct_allowed: wait_window_seconds = max(step_direct_grace_seconds, direct_retry_wait_seconds) wait_deadline = time.time() + wait_window_seconds wait_logged_at = 0.0 attempt = 0 while True: proxy = self.get_proxies( excluded_proxy_keys=excluded_proxy_keys, step_name=step_name, lease_selected=True, ) elapsed_ms = int((time.perf_counter() - started_at) * 1000) if proxy: self._emit_step_trace( domain_name, step_name, "proxy_selected", elapsed_ms=elapsed_ms, proxy_mode="proxy", excluded_proxy_count=len(excluded_proxy_keys), retry_attempts=attempt, ) return proxy if not proxy_enabled: self._emit_step_trace( domain_name, step_name, "proxy_direct_fallback", elapsed_ms=elapsed_ms, proxy_mode="direct", recent_proxy_status=self.proxy_last_refresh_status, ) return None now_ts = time.time() if now_ts >= wait_deadline: break attempt += 1 if not self.proxy_refresh_lock.locked(): self.trigger_proxy_refresh( reason=f"step_wait:{step_name}", reset_cooldown=bool(direct_retry_wait_seconds) or step_name == '注册状态检测', ) if now_ts - wait_logged_at >= 1.0: wait_logged_at = now_ts logger.info( f"{step_name} 暂无可用代理，继续等待补货: {domain_name}, " f"attempt={attempt}, waited_ms={elapsed_ms}, 最近状态: {self.proxy_last_refresh_status}" ) time.sleep(0.05 if direct_retry_wait_seconds > 0 else 0.1) if proxy_enabled and direct_allowed: self._emit_step_trace( domain_name, step_name, "proxy_direct_fallback", elapsed_ms=int((time.perf_counter() - started_at) * 1000), proxy_mode="direct", recent_proxy_status=self.proxy_last_refresh_status, retry_attempts=attempt, ) return None detail = self.proxy_last_refresh_status elapsed_ms = int((time.perf_counter() - started_at) * 1000) logger.error( f"{step_name} 无可用代理，且当前不允许直连兜底: {domain_name}，" f"waited_ms={elapsed_ms}, attempts={attempt}, 最近代理状态: {detail}" ) self._emit_step_trace( domain_name, step_name, "proxy_unavailable", elapsed_ms=elapsed_ms, level='warning', proxy_mode="required", recent_proxy_status=detail, retry_attempts=attempt, ) if not self._is_current_single_step_task(): self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED) return '__NO_PROXY__' def _should_rotate_proxy_on_error(self, proxy, reason, step_name, domain_name): if not proxy or not self.proxy_config.get('proxy_enable', False): return False normalized_reason = str(reason or "") if not self._is_external_dependency_issue(normalized_reason): return False if self._is_proxy_auth_issue(normalized_reason): self._mark_proxy_source_failure(proxy, normalized_reason) self.remove_proxy(proxy) try: self.trigger_proxy_refresh(reason=f"proxy_rotate:{step_name}", reset_cooldown=True) except Exception as refresh_error: logger.debug(f"{step_name} 触发代理快速补货失败: {refresh_error}") logger.warning(f"{step_name} 当前代理异常，切换下一个代理继续: {domain_name}, 原因: {normalized_reason}") return True def _should_retry_external_issue(self, proxy, reason, step_name, domain_name): normalized_reason = str(reason or "") if not self._is_external_dependency_issue(normalized_reason): return False if proxy and self._allow_direct_connection_for_step(step_name): if self._is_proxy_auth_issue(normalized_reason): self._mark_proxy_source_failure(proxy, normalized_reason) self.remove_proxy(proxy) self._set_step_force_direct_once(step_name, True) try: self.trigger_proxy_refresh(reason=f"direct_fallback:{step_name}", reset_cooldown=True) except Exception as refresh_error: logger.debug(f"{step_name} 触发直连兜底前代理补货失败: {refresh_error}") logger.warning( f"{step_name} 代理链路异常，下一次尝试直连兜底: " f"{domain_name}, 原因: {normalized_reason}" ) return True if self._should_rotate_proxy_on_error(proxy, normalized_reason, step_name, domain_name): return True if self.proxy_config.get('proxy_enable', False): if not proxy and self._allow_direct_connection_for_step(step_name): retry_proxy_wait_seconds = self._proxy_direct_retry_wait_for_step(step_name) forced_wait_seconds = max( self._proxy_direct_fallback_grace_for_step(step_name), min( self.proxy_step_wait_timeout_seconds or 0.0, retry_proxy_wait_seconds, ) if self.proxy_step_wait_timeout_seconds > 0 else retry_proxy_wait_seconds, ) self._set_step_proxy_retry_wait(step_name, forced_wait_seconds) try: self.trigger_proxy_refresh(reason=f"external_retry:{step_name}", reset_cooldown=True) except Exception as refresh_error: logger.debug(f"{step_name} 触发代理补货失败: {refresh_error}") logger.warning( f"{step_name} 外部链路异常，已触发代理补货并继续重试: " f"{domain_name}, 原因: {normalized_reason}" ) return True return False def _mark_blacklisted(self, domain_id, domain_name, reason): self._set_task_detect_status(DETECT_STATUS_BLACKLISTED) if self._is_current_single_step_task(): logger.info(f"single_step 域名命中黑名单，跳过本地域名表/黑名单表写入: {domain_name}, 原因: {reason}") return self.db.mark_domain_blacklisted(domain_id, domain_name, reason) logger.info(f"域名已加入黑名单: {domain_name}, 原因: {reason}") def _step_result_payload(self, *, ok, state, message="", **extra): payload = { "status": bool(ok), "state": state, "message": message or "", "checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } payload.update(extra) return payload def _record_step_result(self, domain_id, field_name, *, ok, state, message="", **extra): if not field_name: return payload = self._step_result_payload(ok=ok, state=state, message=message, **extra) if self._is_current_single_step_task(): self._cache_single_step_result_payload(domain_id, field_name, payload) return with self._step_result_cache_lock: self._step_result_cache[(int(domain_id or 0), field_name)] = dict(payload) self._upsert_json_detection(domain_id, **{field_name: payload}) def _should_blacklist_result(self, message): if not message: return False lowered = str(message).lower() keywords = [ "检测到敏感词", "黑名单分类", "风险网站", "高危网站", "clienthold", "serverhold", "拦截", "敏感内容", ] return any(keyword.lower() in lowered for keyword in keywords) def _mark_detection_failed(self, domain_id, domain_name, step_name, reason, field_name=None): reason = reason or "未知错误" if self._is_current_single_step_task(): self._set_task_detect_status(DETECT_STATUS_FAILED) if field_name: self._record_step_result(domain_id, field_name, ok=False, state="failed", message=reason, step=step_name) logger.warning(f"{step_name} single_step 技术失败，跳过本地域名状态写入: {domain_name}, 原因: {reason}") return current_status = self._get_task_detect_status() or 0 if current_status == DETECT_STATUS_BLACKLISTED: logger.warning(f"{step_name} 技术失败，但域名已是黑名单状态，保持黑名单不回退: {domain_name}, 原因: {reason}") else: self._enqueue_domain_status_update(domain_id, DETECT_STATUS_FAILED) self._set_task_detect_status(DETECT_STATUS_FAILED) if field_name: self._record_step_result(domain_id, field_name, ok=False, state="failed", message=reason, step=step_name) logger.warning(f"{step_name} 技术失败，域名标记为检测失败: {domain_name}, 原因: {reason}") def _mark_detection_rejected(self, domain_id, domain_name, step_name, reason, field_name=None): reason = reason or "业务判定未通过" self._set_task_detect_status(DETECT_STATUS_FAILED) if field_name: self._record_step_result(domain_id, field_name, ok=False, state="rejected", message=reason, step=step_name) if self._is_current_single_step_task(): logger.warning(f"{step_name} single_step 业务不通过，跳过本地域名状态写入: {domain_name}, 原因: {reason}") return self._enqueue_domain_status_update(domain_id, DETECT_STATUS_FAILED) logger.warning(f"{step_name} 业务判定未通过，终止后续步骤: {domain_name}, 原因: {reason}") def _mark_detection_degraded(self, domain_id, domain_name, step_name, reason, field_name=None, review_required=True): reason = reason or "外部依赖异常" if field_name: self._record_step_result(domain_id, field_name, ok=False, state="degraded", message=reason, step=step_name) if self._is_current_single_step_task(): logger.warning(f"{step_name} single_step 外部依赖异常，跳过本地 review 状态写入: {domain_name}, 原因: {reason}") return if review_required: try: self._enqueue_review_status_update(domain_id, REVIEW_STATUS_PENDING) except Exception as review_error: logger.warning(f"{step_name} 降级后设置待人工复核失败: {domain_name}, 错误: {review_error}") logger.warning(f"{step_name} 外部依赖异常，步骤降级继续执行: {domain_name}, 原因: {reason}") def _is_external_dependency_issue(self, message): if not message: return False lowered = str(message).lower() keywords = [ "timeout", "timed out", "超时", "请求超时", "连接超时", "读取超时", "网络超时", "请求失败", "网络异常", "连接失败", "连接中断", "连接被重置", "服务暂不可用", "connection", "connection reset", "connection refused", "max retries exceeded", "httpsconnectionpool", "proxyerror", "remotedisconnected", "temporarily unavailable", "502 bad gateway", "503 service unavailable", "504 gateway timeout", "network", "ssl", "read timeout", "connecttimeout", "operation timed out", "connection timed out", "resolving timed out", "http 403", "http 429", "status code 403", "status code 429", "状态码: 403", "状态码: 429", "状态码: 502", "状态码: 503", "状态码: 504", "forbidden", "empty reply from server", "recv failure", "curl: (28)", "curl:(28)", "curl: (52)", "curl:(52)", "响应格式异常", "cannot unpack non-iterable nonetype object", ] return any(keyword in lowered for keyword in keywords) def _resolve_proxy_step_retry_budget(self, step_name): overrides = (getattr(self, "proxy_step_retry_budget_overrides", None) or {}).get(step_name) or {} max_attempts = max( 1, int(overrides.get("max_attempts", getattr(self, "proxy_step_retry_max_attempts", 12)) or 1), ) max_seconds = max( 5.0, float(overrides.get("max_seconds", getattr(self, "proxy_step_retry_max_seconds", 45)) or 5.0), ) return max_attempts, max_seconds def _check_proxy_step_retry_budget(self, step_name, domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=""): elapsed_seconds = max(0.0, time.perf_counter() - float(started_at or 0.0)) max_attempts, max_seconds = self._resolve_proxy_step_retry_budget(step_name) pending_direct_fallback = ( attempt_count < max_attempts and self._allow_direct_connection_for_step(step_name) and self._peek_step_force_direct_once(step_name) ) if attempt_count < max_attempts and (elapsed_seconds < max_seconds or pending_direct_fallback): return False, "" reason = ( f"{step_name} 重试预算耗尽: attempts={attempt_count}, " f"elapsed_ms={int(elapsed_seconds * 1000)}, tried_proxies={len(tried_proxy_keys)}" ) if last_reason: reason = f"{reason}, last_reason={str(last_reason)[:200]}" logger.warning(f"{reason}: {domain_name}") self._emit_step_trace( domain_name, step_name, "retry_budget_exhausted", level='warning', elapsed_ms=int(elapsed_seconds * 1000), retry_attempts=attempt_count, tried_proxy_count=len(tried_proxy_keys), last_reason=str(last_reason or "")[:200], ) return True, reason def _remaining_proxy_step_retry_budget_seconds(self, step_name, started_at, reserve_seconds=0.0): elapsed_seconds = max(0.0, time.perf_counter() - float(started_at or 0.0)) _, max_seconds = self._resolve_proxy_step_retry_budget(step_name) return max(0.0, float(max_seconds or 0.0) - elapsed_seconds - max(0.0, float(reserve_seconds or 0.0))) def _resolve_submit_backlog_limit(self, max_threads, live_active): normalized_threads = max(1, int(max_threads or 1)) live_active = max(0, int(live_active or 0)) configured_backlog_floor = self._runtime_optional_int_override("submit_backlog_floor", "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR") configured_backlog_ceiling = self._runtime_optional_int_override("submit_backlog_ceil", "DOMAINCHECK_SUBMIT_BACKLOG_CEIL") # Default to a full-thread warm window. Large machines should be allowed # to keep enough local backlog to actually fill the configured worker # pool instead of self-throttling to a few hundred runnable items. default_backlog_floor = normalized_threads backlog_floor = max(16, configured_backlog_floor if configured_backlog_floor > 0 else default_backlog_floor) backlog_ceiling = max( backlog_floor, configured_backlog_ceiling if configured_backlog_ceiling > 0 else normalized_threads, ) dynamic_target = max( backlog_floor, live_active + max(32, normalized_threads // 4), ) return min(backlog_ceiling, dynamic_target) def _resolve_dispatch_capacity(self, max_threads, live_active, backlog_limit): normalized_threads = max(1, int(max_threads or 1)) live_active = max(0, int(live_active or 0)) backlog_limit = max(1, int(backlog_limit or 1)) dispatch_cap_multiplier = self._runtime_positive_int_override( "dispatch_cap_multiplier", "DOMAINCHECK_DISPATCH_CAP_MULTIPLIER", default=1, ) dispatch_capacity_cap = max(1, normalized_threads * dispatch_cap_multiplier) return min(dispatch_capacity_cap, max(backlog_limit, live_active + backlog_limit)) def _resolve_claim_batch_size(self, current_limit, available_slots, live_active=None): normalized_limit = max(1, int(current_limit or 1)) available_slots = max(1, int(available_slots or 0)) live_active = max( 0, int(self._get_active_domain_threads() or 0) if live_active is None else int(live_active or 0), ) claim_cap_multiplier = max( 1, int(os.getenv("DOMAINCHECK_CLAIM_CAP_MULTIPLIER", "1") or 1), ) configured_claim_floor = self._runtime_optional_int_override("claim_batch_floor", "DOMAINCHECK_CLAIM_BATCH_FLOOR") configured_claim_ceiling = self._runtime_optional_int_override("claim_batch_ceil", "DOMAINCHECK_CLAIM_BATCH_CEIL") # Default to a full-thread claim window so large worker pools can # actually fill their slots. Operators can still shrink this via env # overrides if they explicitly want a tighter rolling buffer. default_claim_floor = normalized_limit claim_floor = max(4, configured_claim_floor if configured_claim_floor > 0 else default_claim_floor) claim_ceiling = max( claim_floor, configured_claim_ceiling if configured_claim_ceiling > 0 else normalized_limit, ) warm_window = max( claim_floor, live_active + max(32, normalized_limit // 4), ) return max( 1, min( available_slots, normalized_limit * claim_cap_multiplier, max(claim_floor, min(claim_ceiling, warm_window)), ), ) def _should_proactively_top_up_pipeline(self, claimed_count, refill_slots, max_threads): claimed_count = max(0, int(claimed_count or 0)) refill_slots = max(0, int(refill_slots or 0)) normalized_threads = max(1, int(max_threads or 1)) remaining_gap = max(0, refill_slots - claimed_count) if remaining_gap <= 0: return False configured_threshold = int(os.getenv("DOMAINCHECK_PIPELINE_TOPUP_THRESHOLD", "0") or 0) threshold = max( 1, configured_threshold if configured_threshold > 0 else max(128, normalized_threads // 4), ) return remaining_gap >= threshold def _should_prefetch_sync_tasks(self, pending_buffer_count, inflight_count, max_threads): pending_buffer_count = max(0, int(pending_buffer_count or 0)) inflight_count = max(0, int(inflight_count or 0)) normalized_threads = max(1, int(max_threads or 1)) local_work = pending_buffer_count + inflight_count configured_threshold = int(os.getenv("DOMAINCHECK_SYNC_PULL_PREFETCH_THRESHOLD", "0") or 0) threshold = max( 32, configured_threshold if configured_threshold > 0 else max(64, min(normalized_threads, normalized_threads // 2)), ) return local_work <= threshold def _complete_detection(self, domain_id, domain_name, domain): register_status = int((domain or {}).get('register_status', 0) or 0) use_status = int((domain or {}).get('use_status', 0) or 0) expire_date = (domain or {}).get('expire_date') self._enqueue_domain_completion( domain_id, register_status=register_status, use_status=use_status, expire_date=expire_date, ) if use_status == 0 and register_status == REGISTER_STATUS_AVAILABLE and expire_date: logger.info(f"域名 {domain_name} 满足条件，已将expire_date置空") domain['expire_date'] = None if register_status == REGISTER_STATUS_AVAILABLE: logger.info(f"域名 {domain_name} 满足条件，已设置为待人工复核") def _should_run_jucha(self, domain): return self.detect_options.get('detect_jucha', False) and domain.get('jucha_status', 0) != THIRD_PARTY_STATUS_DONE def _should_run_juziseo(self, domain): return self.detect_options.get('detect_juziseo', False) and domain.get('juziseo_status', 0) != THIRD_PARTY_STATUS_DONE def _get_detect_execution_order(self): default_order = [ 'detect_register', 'detect_baidu_site', 'detect_360_site', 'detect_chinaz', 'detect_aizhan', 'detect_wayback', 'detect_jucha', 'detect_juziseo', ] configured_order = self.detect_options.get('detect_order') or [] normalized_order = [key for key in configured_order if key in default_order] for key in default_order: if key not in normalized_order: normalized_order.append(key) free_keys = { 'detect_register', 'detect_baidu_site', 'detect_360_site', 'detect_chinaz', 'detect_aizhan', 'detect_wayback', } paid_keys = {'detect_jucha', 'detect_juziseo'} enabled_order = [key for key in normalized_order if self.detect_options.get(key, False)] free_order = [key for key in enabled_order if key in free_keys] paid_order = [key for key in enabled_order if key in paid_keys] return free_order + paid_order def _run_detect_register(self, domain_id, domain, domain_name): is_ykj = domain.get('source_type', 0) == 1 if is_ykj: logger.info(f"一口价域名跳过注册状态检测: {domain_name}") if self._is_current_single_step_task(): self._cache_single_step_result_payload( domain_id, 'register_status', self._step_result_payload( ok=True, state="passed", message="一口价域名跳过注册状态检测", source_type=domain.get('source_type'), skipped=True, ), ) return True logger.info(f"检测注册状态: {domain_name}") tried_proxy_keys = set() started_at = time.perf_counter() attempt_count = 0 last_reason = "" step_name = '注册状态检测' direct_allowed = self._allow_direct_connection_for_step(step_name) register_single_machine_mode = self._register_single_machine_mode_enabled() direct_first_attempt_limit = 0 # 高并发下如果注册检测仍然直连优先，会先把 controller 本机的外连资源打爆， # 导致大量线程卡在 direct request error，而不是尽快让代理承接流量。 # 这里默认改成“代理优先，直连兜底”；只有显式打开环境变量，或最近已明确 # 进入代理短缺/冷却态时，才允许注册检测直接优先走直连。 register_direct_first = False if direct_allowed: register_direct_first = ( self._env_flag_enabled("DOMAINCHECK_REGISTER_DIRECT_FIRST") or self._proxy_status_indicates_shortage() or register_single_machine_mode ) if register_direct_first and direct_allowed: direct_first_attempt_limit = 1 if register_single_machine_mode and direct_allowed: direct_first_attempt_limit = max( direct_first_attempt_limit, self._register_single_machine_direct_streak_attempts(), ) direct_attempt_count = 0 while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( step_name, domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=last_reason, ) if budget_exhausted: self._mark_detection_failed(domain_id, domain_name, '注册状态检测', budget_reason) return False proxy = None if direct_attempt_count < direct_first_attempt_limit: direct_attempt_count += 1 self._emit_step_trace( domain_name, step_name, 'proxy_direct_fallback', proxy_mode='direct_perf' if register_single_machine_mode else 'direct_first', retry_attempts=attempt_count, ) else: proxy = self._get_proxy_for_step(domain_id, domain_name, step_name, excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 try: tld = domain_name.split('.')[-1] remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( step_name, started_at, reserve_seconds=0.2, ) status, expire_date = register.check_register( domain_name, tld, proxy if proxy else None, budget_seconds=remaining_budget_seconds, ) if status != -1: if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='注册状态检测') normalized_expire_date = expire_date if expire_date: from datetime import datetime, timedelta try: date_part = expire_date.split(' ')[0] expire_date_obj = datetime.strptime(date_part, "%Y-%m-%d") normalized_expire_date = (expire_date_obj + timedelta(days=75)).strftime("%Y-%m-%d") except Exception as e: logger.error(f"处理过期日期失败: {e}") normalized_expire_date = expire_date if status in {REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD}: hold_label = ( "clientHold" if int(status) == int(REGISTER_STATUS_CLIENT_HOLD) else "serverHold" ) blacklist_reason = f"注册状态命中 {hold_label}" if self._is_current_single_step_task(): self._cache_single_step_result_payload( domain_id, 'register_status', self._step_result_payload( ok=False, state="blacklisted", message=blacklist_reason, register_status=int(status), expire_date=normalized_expire_date or "", ), ) else: self.db.update_domain_register_result(domain_id, status, normalized_expire_date) domain['register_status'] = status if normalized_expire_date: domain['expire_date'] = normalized_expire_date self._mark_blacklisted(domain_id, domain_name, blacklist_reason) logger.warning(f"注册状态检测命中黑名单: {domain_name}, 原因: {blacklist_reason}") return False if self._is_current_single_step_task(): self._cache_single_step_result_payload( domain_id, 'register_status', self._step_result_payload( ok=True, state="passed", message="success", register_status=int(status), expire_date=normalized_expire_date or "", ), ) else: self.db.update_domain_register_result(domain_id, status, normalized_expire_date) domain['register_status'] = status if normalized_expire_date: domain['expire_date'] = normalized_expire_date logger.info(f"注册状态检测完成: {domain_name}, 状态: {status}, 过期日期: {expire_date}") return True logger.warning(f"注册状态检测失败，不更新数据: {domain_name}") last_reason = 'register lookup failed' if self._should_rotate_proxy_on_error(proxy, 'register lookup failed', step_name, domain_name): continue if proxy: self.release_proxy(proxy, step_name=step_name, reason='register lookup failed') self._mark_detection_failed(domain_id, domain_name, step_name, '注册状态检测返回未知状态') return False except Exception as e: logger.error(f"注册状态检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if proxy is None and self.proxy_config.get('proxy_enable', False): if direct_allowed and self._proxy_status_indicates_shortage(): self._set_step_force_direct_once(step_name, True) self._set_step_proxy_retry_wait( step_name, self._proxy_direct_fallback_grace_for_step(step_name), ) logger.warning( f"注册状态检测直连失败，代理源仍短缺，下一次继续快速直连: {domain_name}, 原因: {e}" ) else: logger.warning(f"注册状态检测直连失败，切换代理继续: {domain_name}, 原因: {e}") continue if self._should_rotate_proxy_on_error(proxy, str(e), step_name, domain_name): continue if proxy: self.release_proxy(proxy, step_name=step_name, reason=str(e)) self._mark_detection_failed(domain_id, domain_name, step_name, str(e)) return False return True def _run_detect_wayback(self, domain_id, domain_name, sensitive_words, step_payload=None): logger.info(f"检测时光机: {domain_name}") try: normalized_step_payload = dict(step_payload or {}) if isinstance(step_payload, dict) else {} recent_years = normalized_step_payload.get("wayback_recent_years") stop_on_first_hit = normalized_step_payload.get("wayback_stop_on_first_hit") wayback_result = self.wayback_detector.scan_snapshots( domain_name, sensitive_words=sensitive_words, stop_on_first_hit=bool(True if stop_on_first_hit is None else stop_on_first_hit), recent_years=int(recent_years) if str(recent_years or "").strip() else None, ) snapshot_years = wayback_result.get('snapshot_years') or [] is_step_task = self._is_current_single_step_task() backlink_count = int(wayback_result.get('backlink_count', 0) or 0) if not is_step_task: self.db.update_domain_wayback_summary( domain_id, years=",".join(str(year) for year in snapshot_years) if snapshot_years else None, backlink_count=backlink_count, ) self._upsert_json_detection( domain_id, backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')), ) if wayback_result.get('has_sensitive_content'): reason = f"时光机快照命中敏感词: {wayback_result.get('matched_word', '')}".strip() self._record_step_result(domain_id, 'wayback_info', ok=False, state="blacklisted", message=reason) self._mark_blacklisted(domain_id, domain_name, reason) return False if int(wayback_result.get('request_error_count', 0) or 0) > 0 and int(wayback_result.get('checked_snapshot_count', 0) or 0) == 0: reason = "时光机检测失败，快照索引请求异常: " + " | ".join( (wayback_result.get('request_errors') or [])[:3] ) self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info') return True if int(wayback_result.get('failed_snapshot_count', 0) or 0) > 0 and int(wayback_result.get('fetched_snapshot_count', 0) or 0) == 0: reason = ( f"时光机检测失败，未成功抓取任何快照；失败次数 {wayback_result.get('failed_snapshot_count', 0)}" ) self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info') return True self._record_step_result( domain_id, 'wayback_info', ok=True, state="passed", message="success", checked_snapshot_count=int(wayback_result.get('checked_snapshot_count', 0) or 0), fetched_snapshot_count=int(wayback_result.get('fetched_snapshot_count', 0) or 0), failed_snapshot_count=int(wayback_result.get('failed_snapshot_count', 0) or 0), backlink_count=int(wayback_result.get('backlink_count', 0) or 0), backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')), snapshot_years=list(snapshot_years), ) logger.info( "时光机检测完成: %s, 检查快照 %s 个, 成功抓取 %s 个, 失败 %s 个, 最大友链 %s, 耗时 %ss" % ( domain_name, wayback_result.get('checked_snapshot_count', 0), wayback_result.get('fetched_snapshot_count', 0), wayback_result.get('failed_snapshot_count', 0), wayback_result.get('backlink_count', 0), wayback_result.get('elapsed_seconds', 0), ) ) except Exception as e: logger.error(f"时光机检测失败: {domain_name}, 错误: {e}") self._mark_detection_degraded(domain_id, domain_name, '时光机检测', str(e), 'wayback_info') return True return True def _run_detect_chinaz(self, domain_id, domain_name, sensitive_words): logger.info(f"检测站长之家: {domain_name}") tried_proxy_keys = set() started_at = time.perf_counter() attempt_count = 0 last_reason = "" while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( '站长之家检测', domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=last_reason, ) if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', budget_reason, 'chinaz_info') return True if self._should_force_direct_first_attempt('站长之家检测', attempt_count): self._emit_step_trace( domain_name, '站长之家检测', 'proxy_direct_fallback', elapsed_ms=0, proxy_mode='direct', recent_proxy_status=self.proxy_last_refresh_status, retry_attempts=int(attempt_count or 0), reason='single_machine_direct_first', ) proxy = None else: proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 try: remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( '站长之家检测', started_at, reserve_seconds=0.2, ) logger.info( f"站长之家调用边界: domain={domain_name} | attempt={attempt_count} " f"| module={getattr(chinaz, '__file__', 'unknown')} | proxy={'yes' if proxy else 'no'}" ) success, message, seo_data = chinaz.check_title( domain_name, sensitive_words, proxy, budget_seconds=remaining_budget_seconds, ) logger.info( f"站长之家返回边界: domain={domain_name} | attempt={attempt_count} " f"| success={success} | message={message}" ) if not success: logger.warning(f"站长之家检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): if proxy: self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'chinaz_info', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_rotate_proxy_on_error(proxy, message, '站长之家检测', domain_name): continue if self._is_external_dependency_issue(message): if proxy: self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', message or 'external dependency issue', 'chinaz_info') return True if proxy: self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '站长之家检测', message or 'business rejected', 'chinaz_info') return False if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='站长之家检测') self._record_step_result(domain_id, 'chinaz_info', ok=True, state="passed", message=message or "success", seo=seo_data) logger.info(f"站长之家检测完成: {domain_name}") return True except Exception as e: logger.error(f"站长之家检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if self._should_rotate_proxy_on_error(proxy, str(e), '站长之家检测', domain_name): continue if self._is_external_dependency_issue(e): if proxy: self.release_proxy(proxy, step_name='站长之家检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info') return True if proxy: self.release_proxy(proxy, step_name='站长之家检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info') return False return True def _run_detect_aizhan(self, domain_id, domain_name, sensitive_words): logger.info(f"检测爱站网: {domain_name}") tried_proxy_keys = set() started_at = time.perf_counter() attempt_count = 0 last_reason = "" while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( '爱站网检测', domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=last_reason, ) if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', budget_reason, 'aizhan_info') return True if self._should_force_direct_first_attempt('爱站网检测', attempt_count): self._set_step_force_direct_once('爱站网检测', True) proxy = self._get_proxy_for_step(domain_id, domain_name, '爱站网检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 try: remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( '爱站网检测', started_at, reserve_seconds=0.2, ) success, message = aizhan.check_aizhan( domain_name, sensitive_words, proxy, budget_seconds=remaining_budget_seconds, ) if not success: logger.warning(f"爱站网检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'aizhan_info', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_degrade_external_issue_immediately('爱站网检测', message): if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info') return True if self._should_retry_external_issue(proxy, message, '爱站网检测', domain_name): continue if self._is_external_dependency_issue(message): if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info') return True if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '爱站网检测', message or 'business rejected', 'aizhan_info') return False if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='爱站网检测') self._record_step_result(domain_id, 'aizhan_info', ok=True, state="passed", message=message or "success") logger.info(f"爱站网检测完成: {domain_name}") return True except Exception as e: logger.error(f"爱站网检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if self._should_degrade_external_issue_immediately('爱站网检测', e): if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') return True if self._should_retry_external_issue(proxy, str(e), '爱站网检测', domain_name): continue if self._is_external_dependency_issue(e): if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') return True if proxy: self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') return False return True def _run_detect_baidu(self, domain_id, domain_name, sensitive_words): logger.info(f"检测百度: {domain_name}") tried_proxy_keys = set() started_at = time.perf_counter() attempt_count = 0 last_reason = "" while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( '百度site检测', domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=last_reason, ) if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '百度site检测', budget_reason, 'baidu_site') return True if self._should_force_direct_first_attempt('百度site检测', attempt_count): self._set_step_force_direct_once('百度site检测', True) proxy = self._get_proxy_for_step(domain_id, domain_name, '百度site检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 try: remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( '百度site检测', started_at, reserve_seconds=0.2, ) success, message = baidu.check_site( domain_name, sensitive_words, proxy, budget_seconds=remaining_budget_seconds, ) if not success: logger.warning(f"百度site检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): if proxy: self.release_proxy(proxy, step_name='百度site检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_retry_external_issue(proxy, message, '百度site检测', domain_name): continue if self._is_external_dependency_issue(message): if proxy: self.release_proxy(proxy, step_name='百度site检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site') return True if proxy: self.release_proxy(proxy, step_name='百度site检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '百度site检测', message or 'business rejected', 'baidu_site') return False if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='百度site检测') self._record_step_result(domain_id, 'baidu_site', ok=True, state="passed", message=message or "success") logger.info(f"百度site检测完成: {domain_name}") return True except Exception as e: logger.error(f"百度site检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if self._should_retry_external_issue(proxy, str(e), '百度site检测', domain_name): continue if self._is_external_dependency_issue(e): if proxy: self.release_proxy(proxy, step_name='百度site检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '百度site检测', str(e), 'baidu_site') return True if proxy: self.release_proxy(proxy, step_name='百度site检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '百度site检测', str(e), 'baidu_site') return False return True def _run_detect_360(self, domain_id, domain_name, sensitive_words): logger.info(f"检测360: {domain_name}") tried_proxy_keys = set() started_at = time.perf_counter() attempt_count = 0 last_reason = "" while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( '360检测', domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=last_reason, ) if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '360检测', budget_reason, 'qihu360_site') return True proxy = self._get_proxy_for_step(domain_id, domain_name, '360检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 try: remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( '360检测', started_at, reserve_seconds=0.2, ) passed, message = c360.check_domain( domain_name, sensitive_words, proxy, budget_seconds=remaining_budget_seconds, ) if not passed: logger.warning(f"360检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): if proxy: self.release_proxy(proxy, step_name='360检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'qihu360_site', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_retry_external_issue(proxy, message, '360检测', domain_name): continue if self._is_external_dependency_issue(message): if proxy: self.release_proxy(proxy, step_name='360检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '360检测', message or 'external dependency issue', 'qihu360_site') return True if proxy: self.release_proxy(proxy, step_name='360检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '360检测', message or 'business rejected', 'qihu360_site') return False if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='360检测') self._record_step_result(domain_id, 'qihu360_site', ok=True, state="passed", message=message or "success") logger.info(f"360检测完成: {domain_name}") return True except Exception as e: logger.error(f"360检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if self._should_retry_external_issue(proxy, str(e), '360检测', domain_name): continue if self._is_external_dependency_issue(e): if proxy: self.release_proxy(proxy, step_name='360检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '360检测', str(e), 'qihu360_site') return True if proxy: self.release_proxy(proxy, step_name='360检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '360检测', str(e), 'qihu360_site') return False return True def _run_detect_jucha(self, domain_id, domain, domain_name): if not self._should_run_jucha(domain): return True self._init_optional_detectors() if self.jc_instance is None: logger.warning("聚查检测器未初始化，跳过本次聚查检测") return True proxy = self._get_proxy_for_step(domain_id, domain_name, '聚查检测') if proxy == '__NO_PROXY__': return False self.jc_instance.session.proxies = proxy or {} beian_update_payload = {} logger.info(f"检测聚查WHOIS: {domain_name}") try: success, message, whois_info = self.jc_instance.check_whois_domain(domain_name) if not success: logger.warning(f"聚查WHOIS查询失败: {domain_name}, 原因: {message}") else: logger.info(f"WHOIS查询结果: {success}, {message}, {whois_info}") if 'clientHold' in whois_info or 'serverHold' in whois_info: logger.warning(f"聚查WHOIS检测未通过: {domain_name}, 原因: 域名状态包含clientHold或serverHold") self._mark_blacklisted(domain_id, domain_name, '域名状态包含clientHold或serverHold') return False logger.info(f"聚查WHOIS检测完成: {domain_name}") except Exception as e: logger.error(f"聚查WHOIS检测失败: {domain_name}, 错误: {e}") logger.info(f"检测聚查备案: {domain_name}") try: success, message, beian_info = self.jc_instance.beian_check_domain(domain_name) if not success: logger.warning(f"聚查备案查询失败: {domain_name}, 原因: {message}") else: logger.info(f"备案查询结果: {success}, {message}, {beian_info}") if isinstance(beian_info, tuple) and len(beian_info) == 4: beian_time, company_type, website_url, has_beian = beian_info has_beian_flag = 2 if has_beian == '当前存在' else 3 beian_year = None if beian_time: try: beian_year = beian_time.split('-')[0] except Exception: pass beian_update_payload = { "company_type": company_type, "website_url": website_url, "has_beian": has_beian_flag, "beian_year": beian_year, } logger.info(f"聚查备案检测完成: {domain_name}") except Exception as e: logger.error(f"聚查备案检测失败: {domain_name}, 错误: {e}") logger.info(f"检测聚查拦截: {domain_name}") try: success, message, safe_info = self.jc_instance.safe_check_domain(domain_name) if not success: logger.warning(f"聚查安全检测失败: {domain_name}, 原因: {message}") else: logger.info(f"安全检测结果: {success}, {message}, {safe_info}") check_items = ['QQ检测', '微信检测', '抖音检测', '被墙检测', '百度检测', '谷歌检测', '火狐检测'] blacklist_reason = [] if isinstance(safe_info, tuple): for i, item in enumerate(safe_info): if isinstance(item, tuple): if len(item) >= 1 and item[0] == 3 and i < len(check_items): blacklist_reason.append(f"{check_items[i]}: 拦截") elif len(item) >= 2 and item[0] == 2 and '拦截' in item[1] and i < len(check_items): blacklist_reason.append(f"{check_items[i]}: 拦截") if blacklist_reason: reason_str = "；".join(blacklist_reason) logger.warning(f"聚查安全检测未通过: {domain_name}, 原因: {reason_str}") self._mark_blacklisted(domain_id, domain_name, reason_str) return False logger.info(f"聚查拦截检测完成: {domain_name}") except Exception as e: logger.error(f"聚查拦截检测失败: {domain_name}, 错误: {e}") finally: self.db.update_domain_beian_info_and_mark_jucha_detected( domain_id, company_type=beian_update_payload.get("company_type"), website_url=beian_update_payload.get("website_url"), has_beian=beian_update_payload.get("has_beian"), beian_year=beian_update_payload.get("beian_year"), ) if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='聚查检测') return True def _run_detect_juziseo(self, domain_id, domain, domain_name, sensitive_words): if not self._should_run_juziseo(domain): return True self._init_optional_detectors() if self.juziseo_instance is None: logger.warning("桔子SEO检测器未初始化，跳过本次桔子检测") return True proxy = self._get_proxy_for_step(domain_id, domain_name, '桔子检测') if proxy == '__NO_PROXY__': return False self.juziseo_instance.session.proxies = proxy or {} logger.info(f"检测桔子历史: {domain_name}") try: success, message = self.juziseo_instance.check_history(domain_name, sensitive_words) if not success: logger.warning(f"桔子历史检测未通过: {domain_name}, 原因: {message}") self._mark_blacklisted(domain_id, domain_name, message) return False logger.info(f"桔子历史检测完成: {domain_name}") except Exception as e: logger.error(f"桔子历史检测失败: {domain_name}, 错误: {e}") logger.info(f"检测桔子外链: {domain_name}") try: success, message = self.juziseo_instance.check_external_link(domain_name, sensitive_words) if not success: logger.warning(f"桔子外链检测未通过: {domain_name}, 原因: {message}") self._mark_blacklisted(domain_id, domain_name, message) return False logger.info(f"桔子外链检测完成: {domain_name}") except Exception as e: logger.error(f"桔子外链检测失败: {domain_name}, 错误: {e}") finally: self.db.mark_juziseo_detected(domain_id) if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) self.release_proxy(proxy, step_name='桔子检测') return True def _upsert_json_detection(self, domain_id, **payload): existing = self.db.fetch_one("SELECT id FROM domain_detections WHERE domain_id = %s", (domain_id,)) columns = [] params = [] for key, value in payload.items(): columns.append(key) params.append(json.dumps(value) if isinstance(value, (dict, list, tuple)) else value) if not columns: return True if existing: assignments = ", ".join([f"{column} = %s" for column in columns] + ["update_time = CURRENT_TIMESTAMP"]) params.append(domain_id) sql = f"UPDATE domain_detections SET {assignments} WHERE domain_id = %s" else: sql = f"INSERT INTO domain_detections (domain_id, {', '.join(columns)}) VALUES (%s, {', '.join(['%s'] * len(columns))})" params = [domain_id] + params return self.db.execute(sql, tuple(params)) def detect_domain(self, domain_id, domain, job_context=None): """ 检测单个域名 :param domain_id: 域名ID :param domain: 域名对象（包含id、domain、source_type等字段） """ import threading thread_id = threading.current_thread().ident # 从域名对象中获取域名字符串 domain_name = domain.get('domain', '') logger.info(f"线程 {thread_id} 开始检测域名: {domain_name}") job_context = job_context or {} job_item_id = job_context.get('job_item_id') job_id = job_context.get('job_id') claim_token = job_context.get('claim_token') task_mode = str(job_context.get('task_mode') or '').strip() or 'domain_pipeline' step_payload = job_context.get('step_payload') if isinstance(job_context.get('step_payload'), dict) else {} requested_step_code = str(step_payload.get('step_code') or job_context.get('step_code') or '').strip() is_step_task = bool(requested_step_code) cycle_token = str(job_context.get('cycle_token') or self.current_cycle_token or '').strip() job_code = str(job_context.get('job_code') or self.current_job_code or '').strip() session_id = int(job_context.get('session_id') or 0) domain_started_at = time.perf_counter() self._set_current_task_context( domain_id=domain_id, is_step_task=is_step_task, task_mode=task_mode, job_item_id=job_item_id, ) early_abort_reason = self._detection_session_abort_reason(session_id) if early_abort_reason: self._release_job_item_for_session_abort( job_item_id, claim_token, session_id=session_id, domain_name=domain_name, detail="queued_before_start", ) logger.info( f"检测会话已切换，跳过旧域名任务派发: {domain_name}, " f"job_item_id={job_item_id or 0}, reason={early_abort_reason}" ) self._emit_step_trace( domain_name, "domain", "aborted", job_item_id=job_item_id, elapsed_ms=0, ok=False, error=f"检测会话已失效: reason={early_abort_reason} stage=queued_before_start domain={domain_name}"[:300], ) self._clear_current_task_context() return current_active_threads = self._change_active_domain_threads(1) self._note_domain_started() if self.detecting and ( current_active_threads <= 16 or current_active_threads % 10 == 0 or current_active_threads >= max(1, int(getattr(self, "thread_count", 1) or 1)) ): self._mark_detection_phase( "running", f"当前实际线程数量: {current_active_threads}/{self.thread_count}", active_threads=current_active_threads, max_threads=self.thread_count, ) self._emit_step_trace( domain_name, "domain", "started", job_item_id=job_item_id, thread_id=thread_id, cycle_token=cycle_token, job_code=job_code, ) def finalize_job_item(final_status, message="", result_payload=None): self._note_domain_result() if not job_item_id or not claim_token: return abort_reason = self._detection_session_abort_reason(session_id) if abort_reason: self._release_job_item_for_session_abort( job_item_id, claim_token, session_id=session_id, domain_name=domain_name, detail=f"finalize:{final_status}", ) logger.warning( f"检测会话已切换，跳过任务项完成态回写并释放回队列: domain={domain_name}, " f"job_item_id={job_item_id}, status={final_status}, reason={abort_reason}" ) return try: db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db" db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-" event_payload = { "domain_id": domain_id, "domain": domain_name, "status": final_status, "cycle_token": cycle_token, "job_code": job_code, } self._enqueue_job_finalization( job_item_id=job_item_id, claim_token=claim_token, final_status=final_status, message=message or final_status, result_payload=result_payload, job_id=job_id, node_code=config.NODE_CODE, event_type=f"domain_{final_status}", event_level='error' if final_status == 'failed' else 'info', event_message=message or f"{domain_name} -> {final_status}", event_payload=event_payload, ) logger.info( "登记 domain 终态事件入队: " f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} " f"event_type=domain_{final_status} db={db_host}:{db_name}" ) except Exception as finalize_error: logger.warning( f"回写任务项状态失败: {domain_name}, 错误: {finalize_error}, " f"job_id={job_id or 0}, job_item_id={job_item_id or 0}, " f"event_type=domain_{final_status}" ) def renew_job_item_lease(detail=""): if is_step_task or not job_item_id or not claim_token: return abort_reason = self._detection_session_abort_reason(session_id) if abort_reason: self._release_job_item_for_session_abort( job_item_id, claim_token, session_id=session_id, domain_name=domain_name, detail=f"renew:{detail}", ) logger.warning( f"检测会话已切换，跳过任务项续租并释放回队列: domain={domain_name}, " f"job_item_id={job_item_id}, detail={detail}, reason={abort_reason}" ) return try: self.db.renew_detect_job_item_lease( job_item_id, claim_token, # 租约续期必须和线程数解耦；线程数越高不代表单任务应该锁更久， # 否则节点重启或线程异常后会把大量 running 项锁死数天。 lease_seconds=900, ) except Exception as renew_error: logger.warning(f"续租任务项失败: {domain_name}, 错误: {renew_error}") try: # 检查是否需要停止 if not self.running: logger.info(f"检测已停止，跳过域名: {domain_name}") finalize_job_item('failed', 'Worker 已停止，任务未执行') return self._assert_detection_session_active(session_id, domain_name=domain_name, stage="prepare") logger.info(f"开始检测域名: {domain_name}") if not is_step_task: self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING) self._set_task_detect_status(DETECT_STATUS_RUNNING) if job_item_id and claim_token: self._enqueue_running_mark(job_item_id, claim_token) renew_job_item_lease("domain_started") db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db" db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-" self.db.append_detect_run_event( job_id, job_item_id, config.NODE_CODE, event_type='domain_started', message=f"开始检测域名: {domain_name}", payload={ "domain_id": domain_id, "domain": domain_name, "cycle_token": cycle_token, "job_code": job_code, "task_mode": task_mode, "step_code": requested_step_code if is_step_task else "", }, ) logger.info( "写入 domain_started 事件成功: " f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} " f"event_type=domain_started db={db_host}:{db_name} " f"task_mode={task_mode} step_code={requested_step_code if is_step_task else '-'}" ) self._sync_worker_log_event( f"开始检测域名: {domain_name}", payload={ "domain_id": domain_id, "domain": domain_name, }, mode='full', job_item_id=job_item_id, ) # 使用共享的敏感词列表 sensitive_words = self.sensitive_words execution_order = self._get_detect_execution_order() parallel_free_steps_enabled = bool( int(os.getenv("DOMAINCHECK_PARALLEL_FREE_STEPS", "1") or 1) ) parallel_free_step_keys = ( 'detect_baidu_site', 'detect_360_site', 'detect_chinaz', 'detect_aizhan', ) parallel_free_step_set = set(parallel_free_step_keys) parallel_free_step_workers = max( 1, int(os.getenv("DOMAINCHECK_PARALLEL_FREE_STEP_WORKERS", "1") or 1), ) def execute_detect_step(detect_key): step_name = self._step_display_name(detect_key) self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:prepare") renew_job_item_lease(detect_key) step_started_at = time.perf_counter() self._emit_step_trace( domain_name, step_name, "started", job_item_id=job_item_id, detect_key=detect_key, ) step_ok = True if detect_key == 'detect_register': step_ok = self._run_detect_register(domain_id, domain, domain_name) elif detect_key == 'detect_baidu_site': step_ok = self._run_detect_baidu(domain_id, domain_name, sensitive_words) elif detect_key == 'detect_360_site': step_ok = self._run_detect_360(domain_id, domain_name, sensitive_words) elif detect_key == 'detect_chinaz': step_ok = self._run_detect_chinaz(domain_id, domain_name, sensitive_words) elif detect_key == 'detect_aizhan': step_ok = self._run_detect_aizhan(domain_id, domain_name, sensitive_words) elif detect_key == 'detect_wayback': step_ok = self._run_detect_wayback( domain_id, domain_name, sensitive_words, step_payload=step_payload, ) elif detect_key == 'detect_jucha': step_ok = self._run_detect_jucha(domain_id, domain, domain_name) elif detect_key == 'detect_juziseo': step_ok = self._run_detect_juziseo(domain_id, domain, domain_name, sensitive_words) else: logger.error(f"不支持的检测步骤: {detect_key}") step_ok = False step_elapsed_ms = int((time.perf_counter() - step_started_at) * 1000) self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finished") self._emit_step_trace( domain_name, step_name, "finished", job_item_id=job_item_id, detect_key=detect_key, elapsed_ms=step_elapsed_ms, ok=step_ok, ) return { "detect_key": detect_key, "step_name": step_name, "step_ok": bool(step_ok), "step_elapsed_ms": step_elapsed_ms, } if is_step_task: self._emit_step_trace( domain_name, "domain", "plan_ready", job_item_id=job_item_id, step_count=1, detect_order=requested_step_code, sensitive_word_count=len(sensitive_words), task_mode=task_mode, ) step_result = execute_detect_step(requested_step_code) self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{requested_step_code}:finalize") result_payload = self._build_single_step_job_result( domain_id=domain_id, detect_key=requested_step_code, step_ok=step_result["step_ok"], step_name=step_result["step_name"], ) final_status, final_message = self._resolve_single_step_finalization(result_payload) self._emit_step_trace( domain_name, step_result["step_name"], "single_step_finalized", job_item_id=job_item_id, detect_key=requested_step_code, elapsed_ms=step_result["step_elapsed_ms"], ok=final_status in {"completed", "blacklisted"}, final_status=final_status, result_state=result_payload.get("state"), ) finalize_job_item(final_status, final_message, result_payload=result_payload) return self._emit_step_trace( domain_name, "domain", "plan_ready", job_item_id=job_item_id, step_count=len(execution_order), detect_order=",".join(execution_order), sensitive_word_count=len(sensitive_words), task_mode=task_mode, ) step_index = 0 while step_index < len(execution_order): detect_key = execution_order[step_index] if parallel_free_steps_enabled and detect_key in parallel_free_step_set: parallel_group = [] while step_index < len(execution_order) and execution_order[step_index] in parallel_free_step_set: parallel_group.append(execution_order[step_index]) step_index += 1 if len(parallel_group) > 1 and parallel_free_step_workers > 1: parallel_message = ( f"域名步骤并行组启动: domain={domain_name} | " f"steps={','.join(parallel_group)} | workers={min(parallel_free_step_workers, len(parallel_group))}" ) logger.info(parallel_message) self._sync_worker_log_event( parallel_message, payload={ "domain_id": domain_id, "domain": domain_name, "steps": parallel_group, "workers": min(parallel_free_step_workers, len(parallel_group)), }, mode='full', job_item_id=job_item_id, ) parallel_results = {} with ThreadPoolExecutor(max_workers=min(parallel_free_step_workers, len(parallel_group))) as executor: future_map = {executor.submit(execute_detect_step, key): key for key in parallel_group} for future in as_completed(future_map): key = future_map[future] parallel_results[key] = future.result() for ordered_key in parallel_group: step_result = parallel_results.get(ordered_key) or { "detect_key": ordered_key, "step_name": self._step_display_name(ordered_key), "step_ok": False, "step_elapsed_ms": 0, } self._assert_detection_session_active( session_id, domain_name=domain_name, stage=f"{ordered_key}:parallel_finalize", ) if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( domain_name, step_result["step_name"], "aborted", job_item_id=job_item_id, detect_key=ordered_key, elapsed_ms=step_result["step_elapsed_ms"], ok=False, final_status=final_status, ) failure_reason_map = { 'detect_register': '注册状态检测未通过', 'detect_baidu_site': '百度检测未通过', 'detect_360_site': '360检测未通过', 'detect_chinaz': '站长之家检测未通过', 'detect_aizhan': '爱站检测未通过', 'detect_wayback': '时光机检测未通过', 'detect_jucha': '聚查检测未通过', 'detect_juziseo': '桔子检测未通过', } finalize_job_item(final_status, failure_reason_map.get(ordered_key, f'{step_result["step_name"]} 未通过')) return continue if len(parallel_group) > 1: serial_message = ( f"域名步骤串行组推进: domain={domain_name} | " f"steps={','.join(parallel_group)} | workers=1" ) logger.info(serial_message) self._sync_worker_log_event( serial_message, payload={ "domain_id": domain_id, "domain": domain_name, "steps": parallel_group, "workers": 1, }, mode='full', job_item_id=job_item_id, ) for ordered_key in parallel_group: step_result = execute_detect_step(ordered_key) self._assert_detection_session_active( session_id, domain_name=domain_name, stage=f"{ordered_key}:serial_finalize", ) if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( domain_name, step_result["step_name"], "aborted", job_item_id=job_item_id, detect_key=ordered_key, elapsed_ms=step_result["step_elapsed_ms"], ok=False, final_status=final_status, ) failure_reason_map = { 'detect_register': '注册状态检测未通过', 'detect_baidu_site': '百度检测未通过', 'detect_360_site': '360检测未通过', 'detect_chinaz': '站长之家检测未通过', 'detect_aizhan': '爱站检测未通过', 'detect_wayback': '时光机检测未通过', 'detect_jucha': '聚查检测未通过', 'detect_juziseo': '桔子检测未通过', } finalize_job_item(final_status, failure_reason_map.get(ordered_key, f'{step_result["step_name"]} 未通过')) return continue step_index += 1 step_result = execute_detect_step(detect_key) self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finalize") if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( domain_name, step_result["step_name"], "aborted", job_item_id=job_item_id, detect_key=detect_key, elapsed_ms=step_result["step_elapsed_ms"], ok=False, final_status=final_status, ) failure_reason_map = { 'detect_register': '注册状态检测未通过', 'detect_baidu_site': '百度检测未通过', 'detect_360_site': '360检测未通过', 'detect_chinaz': '站长之家检测未通过', 'detect_aizhan': '爱站检测未通过', 'detect_wayback': '时光机检测未通过', 'detect_jucha': '聚查检测未通过', 'detect_juziseo': '桔子检测未通过', } finalize_job_item(final_status, failure_reason_map.get(detect_key, f'{step_result["step_name"]} 未通过')) return self._assert_detection_session_active(session_id, domain_name=domain_name, stage="completion") completion_tail_started_at = time.perf_counter() complete_detection_started_at = time.perf_counter() self._complete_detection(domain_id, domain_name, domain) complete_detection_elapsed_ms = int((time.perf_counter() - complete_detection_started_at) * 1000) self._set_task_detect_status(DETECT_STATUS_COMPLETED) finalize_enqueue_started_at = time.perf_counter() finalize_job_item('completed', f"域名检测完成: {domain_name}") finalize_enqueue_elapsed_ms = int((time.perf_counter() - finalize_enqueue_started_at) * 1000) trace_emit_started_at = time.perf_counter() if str(os.getenv("DOMAINCHECK_DOMAIN_COMPLETED_TRACE", "0") or "0").strip().lower() not in {"", "0", "false", "no", "off"}: self._emit_step_trace( domain_name, "domain", "completed", job_item_id=job_item_id, elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000), ok=True, sync_mode='off', ) trace_emit_elapsed_ms = int((time.perf_counter() - trace_emit_started_at) * 1000) logger.info(f"域名检测完成: {domain_name}") completion_log_sync_started_at = time.perf_counter() self._sync_worker_log_event( f"域名检测完成: {domain_name}", payload={ "domain_id": domain_id, "domain": domain_name, "status": "completed", }, mode='full', job_item_id=job_item_id, ) completion_log_sync_elapsed_ms = int((time.perf_counter() - completion_log_sync_started_at) * 1000) completion_tail_elapsed_ms = int((time.perf_counter() - completion_tail_started_at) * 1000) if ( completion_tail_elapsed_ms >= 20 or complete_detection_elapsed_ms >= 10 or trace_emit_elapsed_ms >= 10 or completion_log_sync_elapsed_ms >= 10 ): logger.info( f"域名完成尾部耗时: domain={domain_name} | total_ms={completion_tail_elapsed_ms} " f"| complete_detection_ms={complete_detection_elapsed_ms} " f"| finalize_enqueue_ms={finalize_enqueue_elapsed_ms} " f"| trace_emit_ms={trace_emit_elapsed_ms} " f"| completion_log_enqueue_ms={completion_log_sync_elapsed_ms}" ) except RuntimeError as e: if str(e).startswith("检测会话已失效:"): self._release_job_item_for_session_abort( job_item_id, claim_token, session_id=session_id, domain_name=domain_name, detail=str(e), ) logger.info(f"检测会话切换，终止旧域名任务: {domain_name}, 详情: {e}") self._emit_step_trace( domain_name, "domain", "aborted", job_item_id=job_item_id, elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000), ok=False, error=str(e)[:300], ) return raise except Exception as e: logger.error(f"检测域名出错: {domain_name}, 错误: {e}") if not is_step_task: self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED) self._set_task_detect_status(DETECT_STATUS_FAILED) finalize_job_item('failed', str(e)) self._emit_step_trace( domain_name, "domain", "failed", job_item_id=job_item_id, elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000), ok=False, error=str(e)[:300], ) self._sync_worker_log_event( f"检测域名出错: {domain_name}, 错误: {e}", level='error', payload={ "domain_id": domain_id, "domain": domain_name, "status": "failed", }, mode='full', job_item_id=job_item_id, ) finally: self._clear_current_task_context() remaining_active_threads = self._change_active_domain_threads(-1) if self.detecting: self._mark_detection_phase( "running", f"当前实际线程数量: {remaining_active_threads}/{self.thread_count}", active_threads=remaining_active_threads, max_threads=self.thread_count, ) def start_detection(self, session_id: int = 0): """ 开始检测 """ logger.info("开始执行域名检测任务") self.detecting = True self.stop_requested = False total_processed = 0 final_phase = "idle" final_detail = "Worker 等待下一次启动" self._mark_detection_phase("preparing", "开始执行域名检测任务，正在加载配置") self._sync_worker_log_event("开始执行域名检测任务，正在加载配置", mode='key') # 重新加载配置，确保获取最新的配置 try: self.detect_options = self.load_detect_options() self.proxy_config = self.load_proxy_config() self.thread_count = self.load_thread_count() self._last_thread_count_refresh_at = time.time() self.proxy_max_reuse_count = max(8, self.thread_count * 4) logger.info(f"检测线程数设置为: {self.thread_count}") self.load_cookies_from_remote() self._init_optional_detectors() recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED) if recycled: logger.warning(f"检测启动前回收了 {recycled} 个遗留的检测中任务，已标记为检测失败待重试") released_job_items = self._maybe_release_node_job_items(reason="start_detection") if released_job_items: logger.warning(f"检测启动前释放了 {released_job_items} 个当前节点遗留任务项，已重新回到 pending") except Exception as e: logger.error(f"加载配置失败: {e}") import traceback logger.error(traceback.format_exc()) self._mark_detection_phase("failed", f"加载配置失败: {e}") self._sync_worker_log_event(f"加载配置失败: {e}", level='error', mode='key') final_phase = "failed" final_detail = f"加载配置失败: {e}" return # 重新加载cookies self.load_cookies_from_remote() # 重新加载JC实例的cookies if self.detect_options.get('detect_jucha') and self.jc_instance is not None: self.jc_instance.load_cookies() self.jc_instance.load_juming_cookies() # 重新加载Juziseo实例的cookies if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None: self.juziseo_instance.load_cookies() # 刷新代理池 if self.proxy_config.get('proxy_enable', False): logger.info("开始检测，刷新代理池") self._mark_detection_phase("refreshing_proxy", "开始检测，正在刷新代理池") self._sync_worker_log_event("开始检测，正在刷新代理池", mode='key') proxy_ready, proxy_count, proxy_ready_reason = self._prepare_proxy_pool_for_detection_start() if proxy_ready: logger.info( f"检测启动前代理池已就绪，跳过同步刷新: " f"count={proxy_count}, reason={proxy_ready_reason}" ) self._update_runtime_state( "running", f"检测启动前代理池已就绪，直接进入任务派发: count={proxy_count}, reason={proxy_ready_reason}", ) self._sync_worker_log_event( "检测启动前代理池已就绪，直接进入任务派发", payload={"proxy_count": proxy_count, "reason": proxy_ready_reason}, mode='full', ) elif self.proxy_refresh_lock.locked(): if self._wait_for_proxy_refresh_settle(timeout_seconds=2.0): logger.info("检测启动前等待到了可用代理，直接复用当前代理池") else: logger.info("检测启动前代理池仍未就绪，继续主动刷新一次") self.refresh_proxy_pool() else: self.refresh_proxy_pool() # 更新JC和Juziseo实例的代理设置 if self.detect_options.get('detect_jucha') and self.jc_instance is not None: self.jc_instance.proxies = self.get_proxies() if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None: self.juziseo_instance.proxies = self.get_proxies() # 重新加载敏感词，确保获取最新的敏感词列表 try: self.sensitive_words = self._load_sensitive_words_runtime() logger.info(f"重新加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词") except Exception as e: logger.error(f"重新加载敏感词失败: {e}") import traceback logger.error(traceback.format_exc()) self.sensitive_words = [] # 更新配置标签 self.update_config_labels() try: runtime_mode_summary = { "qt_platform": str(os.environ.get("QT_QPA_PLATFORM", "") or "").strip() or "desktop", "worker_mode": str(os.environ.get("WORKER_MODE", "") or "").strip() or "default", "redis_control": bool(self.use_redis), "log_sync_mode": self._worker_log_sync_mode(), "step_trace_enabled": self._worker_step_trace_enabled(), } runtime_mode_message = ( "检测运行模式摘要: " f"core=detect_worker.py | qt_platform={runtime_mode_summary['qt_platform']} | " f"worker_mode={runtime_mode_summary['worker_mode']} | redis_control={1 if runtime_mode_summary['redis_control'] else 0} | " f"log_sync_mode={runtime_mode_summary['log_sync_mode']} | " f"step_trace_enabled={1 if runtime_mode_summary['step_trace_enabled'] else 0}" ) logger.info(runtime_mode_message) self._sync_worker_log_event(runtime_mode_message, payload=runtime_mode_summary, mode='full') pending_domains = collections.deque() inflight_futures = set() legacy_tail_reached = False last_thread_progress_emit_count = 0 last_thread_progress_emit_at = 0.0 last_job_status_refresh_at = 0.0 last_expired_job_recycle_at = 0.0 last_future_fallback_scan_at = 0.0 thread_count_refresh_interval = max( 0.5, float(os.getenv("DOMAINCHECK_THREAD_COUNT_REFRESH_INTERVAL", "2.0") or 2.0), ) job_status_refresh_interval = max( 5.0, float(os.getenv("DOMAINCHECK_JOB_STATUS_REFRESH_INTERVAL", "15.0") or 15.0), ) worker_pool_size = max( 1, int( self.refresh_thread_count_runtime( force=True, min_interval=thread_count_refresh_interval, ) or 1 ), ) configured_thread_stack_size = _configure_worker_thread_stack_size() worker_pool = ThreadPoolExecutor( max_workers=worker_pool_size, thread_name_prefix="detect-domain", ) last_worker_pool_size = worker_pool_size if configured_thread_stack_size > 0: logger.info( f"检测线程池初始化: max_workers={worker_pool_size}, thread_stack_size={configured_thread_stack_size}" ) def prune_finished_futures(): nonlocal inflight_futures, last_future_fallback_scan_at if not inflight_futures: return 0 finished = [] with self._completed_future_lock: while self._completed_futures and len(finished) < 4096: finished.append(self._completed_futures.popleft()) if not finished: now_ts = time.time() if now_ts - last_future_fallback_scan_at < 0.5: return 0 last_future_fallback_scan_at = now_ts finished = [future for future in list(inflight_futures) if future.done()] if not finished: return 0 pruned_count = 0 for future in dict.fromkeys(finished): if future not in inflight_futures: continue inflight_futures.discard(future) pruned_count += 1 try: future.result() except Exception as future_error: logger.error(f"检测任务 future 执行异常: {future_error}") import traceback logger.error(traceback.format_exc()) return pruned_count def emit_thread_progress(max_threads, *, batch_size=0): nonlocal last_thread_progress_emit_count, last_thread_progress_emit_at current_active = max(len(inflight_futures), self._get_active_domain_threads()) should_emit_progress, emit_now = self._should_emit_thread_progress( current_active, max_threads, last_thread_progress_emit_count, last_thread_progress_emit_at, ) if should_emit_progress: logger.info(f"当前实际线程数量: {current_active}/{max_threads}") self._mark_detection_phase( "running", f"当前实际线程数量: {current_active}/{max_threads}", active_threads=current_active, max_threads=max_threads, batch_size=batch_size, processed=total_processed, pending_buffer=len(pending_domains), ) last_thread_progress_emit_count = current_active last_thread_progress_emit_at = emit_now if self.detect_thread: try: self.detect_thread.thread_count_signal.emit(current_active, max_threads) except Exception as e: logger.error(f"发送线程数量更新信号失败: {e}") import traceback logger.error(traceback.format_exc()) def inflight_submission_metrics(max_threads): live_active = max(0, int(self._get_active_domain_threads() or 0)) submitted_total = len(inflight_futures) queued_backlog = max(0, submitted_total - live_active) backlog_limit = self._resolve_submit_backlog_limit(max_threads, live_active) dispatch_capacity = self._resolve_dispatch_capacity(max_threads, live_active, backlog_limit) effective_available = max(0, dispatch_capacity - submitted_total) return live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, effective_available def sync_worker_pool_size(desired_threads): nonlocal last_worker_pool_size normalized_threads = max(1, int(desired_threads or 1)) current_pool_size = max(1, int(getattr(worker_pool, "_max_workers", normalized_threads) or normalized_threads)) if current_pool_size != normalized_threads: worker_pool._max_workers = normalized_threads logger.info(f"动态调整 worker_pool 大小: {current_pool_size} -> {normalized_threads}") self._sync_worker_log_event( f"动态调整 worker_pool 大小: {current_pool_size} -> {normalized_threads}", payload={ "previous_max_workers": current_pool_size, "current_max_workers": normalized_threads, }, mode='full', ) last_worker_pool_size = normalized_threads elif last_worker_pool_size != normalized_threads: last_worker_pool_size = normalized_threads return normalized_threads def append_pending_domains(domains, using_job_queue, queue_label, claim_elapsed_ms, legacy_fetch_elapsed_ms, thread_limit, claim_batch_size, claim_lease_seconds): current_batch_size = len(domains) if current_batch_size <= 0: return for domain in domains: pending_domains.append( { "domain": domain, "using_job_queue": bool(using_job_queue), } ) fetch_trace_payload = { "queue_label": queue_label, "batch_size": current_batch_size, "claim_elapsed_ms": claim_elapsed_ms, "legacy_fetch_elapsed_ms": legacy_fetch_elapsed_ms, "thread_limit": thread_limit, "claim_batch_size": claim_batch_size, "claim_lease_seconds": claim_lease_seconds, "pending_buffer": len(pending_domains), } fetch_trace_message = ( f"批次领取跟踪: queue={queue_label} | batch_size={current_batch_size} | " f"claim_elapsed_ms={claim_elapsed_ms} | legacy_fetch_elapsed_ms={legacy_fetch_elapsed_ms} | " f"thread_limit={thread_limit} | claim_batch_size={claim_batch_size} | " f"claim_lease_seconds={claim_lease_seconds} | pending_buffer={len(pending_domains)}" ) logger.info(fetch_trace_message) self._sync_worker_log_event(fetch_trace_message, payload=fetch_trace_payload, mode='full') self._mark_detection_phase( "fetching", f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名", batch_size=current_batch_size, queue_source="detect_job_items" if using_job_queue else "domains", pending_buffer=len(pending_domains), ) self._sync_worker_log_event( f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名", payload={ "batch_size": current_batch_size, "queue_source": "detect_job_items" if using_job_queue else "domains", "pending_buffer": len(pending_domains), }, mode='full', ) def compute_claim_batch_size(current_limit, available_slots): return self._resolve_claim_batch_size(current_limit, available_slots) last_dispatchable_step_items_check_at = 0.0 dispatchable_step_items_cached = False legacy_domain_fallback_enabled = bool( int(os.getenv("DOMAINCHECK_ENABLE_LEGACY_DOMAIN_FALLBACK", "0") or 0) ) last_legacy_fallback_skip_log_at = 0.0 last_proactive_pipeline_topup_at = 0.0 last_proactive_sync_topup_at = 0.0 def has_dispatchable_step_items(): nonlocal last_dispatchable_step_items_check_at, dispatchable_step_items_cached now_ts = time.time() if now_ts - float(last_dispatchable_step_items_check_at or 0.0) < 1.0: return dispatchable_step_items_cached try: dispatchable_step_items_cached = bool(self.db.has_dispatchable_detect_job_items()) except Exception as e: logger.debug(f"检查标准步骤待派发任务失败: {e}") last_dispatchable_step_items_check_at = now_ts return dispatchable_step_items_cached def start_domain_thread(domain, using_job_queue, max_threads): nonlocal total_processed domain_id = domain['id'] try: job_context = { 'cycle_token': self.current_cycle_token, 'job_code': domain.get('job_code') or self.current_job_code, 'session_id': session_id, } if using_job_queue: job_context.update( { 'job_item_id': domain.get('job_item_id'), 'job_id': domain.get('job_id'), 'claim_token': domain.get('claim_token'), 'task_mode': domain.get('task_mode'), 'step_code': domain.get('step_code'), 'step_payload': domain.get('step_payload'), } ) future = worker_pool.submit(self.detect_domain, domain_id, domain, job_context) future.add_done_callback(self._enqueue_completed_future) inflight_futures.add(future) total_processed += 1 if self.detect_thread: try: self.detect_thread.progress_signal.emit(total_processed, total_processed + len(pending_domains) + len(inflight_futures)) except Exception as e: logger.error(f"发送进度更新信号失败: {e}") import traceback logger.error(traceback.format_exc()) emit_thread_progress(max_threads, batch_size=len(pending_domains) + len(inflight_futures)) return True except Exception as e: logger.error(f"创建或启动线程失败: {e}") import traceback logger.error(traceback.format_exc()) return False try: while self.running: abort_reason = self._detection_session_abort_reason(session_id) if abort_reason: logger.warning(f"检测会话已切换，停止旧会话主循环: {abort_reason}") self._mark_detection_phase("restarting", f"检测会话已切换，停止旧会话主循环: {abort_reason}") self._sync_worker_log_event( f"检测会话已切换，停止旧会话主循环: {abort_reason}", level='warning', mode='key', ) final_phase = "restarting" final_detail = f"检测会话已切换，停止旧会话主循环: {abort_reason}" break if self.stop_requested: logger.info("检测任务收到停止请求，停止继续领取任务") self._mark_detection_phase("stopping", "检测任务收到停止请求，准备安全退出") self._sync_worker_log_event("检测任务收到停止请求，准备安全退出", level='warning', mode='key') final_phase = "stopped" final_detail = "检测任务收到停止请求，已停止继续领取任务" break prune_finished_futures() self._flush_pending_running_marks() self._flush_pending_job_releases() self._flush_pending_job_finalizations() self._flush_pending_domain_completions() self._flush_pending_domain_status_updates() self._flush_pending_review_status_updates() current_thread_limit = self.refresh_thread_count_runtime( min_interval=thread_count_refresh_interval ) max_threads = sync_worker_pool_size(current_thread_limit) live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, available_slots = inflight_submission_metrics(max_threads) now_ts = time.time() if now_ts - last_expired_job_recycle_at >= 2.0: recycle_lock_owner = self._acquire_shared_job_maintenance_lock( "recycle-expired-job-items", ttl_seconds=10, ) if recycle_lock_owner: try: recycled_job_items = self.db.recycle_expired_detect_job_items() last_expired_job_recycle_at = now_ts if recycled_job_items: logger.warning(f"已回收 {recycled_job_items} 个租约过期的任务项，重新回到 pending") finally: self._release_shared_job_maintenance_lock( "recycle-expired-job-items", recycle_lock_owner, ) if self.current_job_id and now_ts - last_job_status_refresh_at >= job_status_refresh_interval: self._log_job_status_refresh_probe(self.current_job_id) self.db.refresh_detect_job_status(self.current_job_id) last_job_status_refresh_at = now_ts pending_buffer_cap_multiplier = self._runtime_positive_int_override( "pending_buffer_cap_multiplier", "DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER", default=1, ) pending_buffer_target = min( max(4, backlog_limit), max(4, int(max_threads or 0) * pending_buffer_cap_multiplier), ) single_step_session_active = self._is_single_step_session_active() refill_slots = max(0, min(int(available_slots or 0), pending_buffer_target - len(pending_domains))) if refill_slots > 0: claim_batch_size = compute_claim_batch_size(max_threads, refill_slots) claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) # single_step 必须绑定当前 job；domain_pipeline 则持续从全局任务队列补位， # 否则旧 job 未完全结束时，新 job 会长期堆在 pending。 scoped_job_id = self._current_scoped_claim_job_id() if scoped_job_id not in (None, "", 0, "0"): self._log_explicit_scope_probe( scoped_job_id=scoped_job_id, max_threads=max_threads, refill_slots=refill_slots, claim_batch_size=claim_batch_size, pending_buffer=len(pending_domains), ) claim_started_at = time.perf_counter() domains = self._claim_detect_job_items( limit=claim_batch_size, lease_seconds=claim_lease_seconds, job_id=scoped_job_id, ) claim_elapsed_ms = int((time.perf_counter() - claim_started_at) * 1000) legacy_fetch_elapsed_ms = 0 proactive_topup_needed = ( not single_step_session_active and self._should_proactively_top_up_pipeline(len(domains), refill_slots, max_threads) ) if domains: append_pending_domains( domains, True, "任务队列", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) if proactive_topup_needed: proactive_now = time.time() if proactive_now - float(last_proactive_pipeline_topup_at or 0.0) >= 0.5: pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads) last_proactive_pipeline_topup_at = proactive_now if pipeline_domains: append_pending_domains( pipeline_domains, True, "任务队列(pipeline推进补位)", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) elif proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0: refill_domains = self._pull_sync_tasks_until_available( thread_limit=max_threads, claim_after_pull=False, ) last_proactive_sync_topup_at = proactive_now if refill_domains: append_pending_domains( refill_domains, True, "任务队列(主动补货补位)", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) if self._should_prefetch_sync_tasks(len(pending_domains), len(inflight_futures), max_threads): proactive_now = time.time() if proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0: self._pull_sync_tasks_until_available( thread_limit=max_threads, claim_after_pull=False, ) last_proactive_sync_topup_at = proactive_now elif not single_step_session_active: pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads) if pipeline_domains: append_pending_domains( pipeline_domains, True, "任务队列(pipeline推进)", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) else: refill_domains = self._pull_sync_tasks_until_available(thread_limit=max_threads) if refill_domains: append_pending_domains( refill_domains, True, "任务队列(主动补货)", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) elif has_dispatchable_step_items(): logger.info("标准步骤队列仍有待派发任务，跳过兼容旧链路补位") elif legacy_domain_fallback_enabled and not legacy_tail_reached: legacy_fetch_limit = max(1, claim_batch_size) legacy_fetch_started_at = time.perf_counter() legacy_domains = self.db.get_domains_to_detect(limit=legacy_fetch_limit, detect_options=self.detect_options) legacy_fetch_elapsed_ms = int((time.perf_counter() - legacy_fetch_started_at) * 1000) if legacy_domains: append_pending_domains( legacy_domains, False, "兼容旧链路", claim_elapsed_ms, legacy_fetch_elapsed_ms, dispatch_capacity, claim_batch_size, claim_lease_seconds, ) if len(legacy_domains) < legacy_fetch_limit: legacy_tail_reached = True elif not legacy_domain_fallback_enabled: current_ts = time.time() if current_ts - float(last_legacy_fallback_skip_log_at or 0.0) >= 5.0: logger.info("兼容旧链路补位已禁用，当前仅允许 detect_job_items 标准步骤补位") last_legacy_fallback_skip_log_at = current_ts else: logger.info("single_step 会话进行中，跳过 pipeline 推进 / 主动补货 / 兼容旧链路补位") dispatched_this_round = 0 while pending_domains: live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, effective_available = inflight_submission_metrics(max_threads) if effective_available <= 0: break if not self.running or self.stop_requested: logger.info("检测已停止，停止创建新线程") self._mark_detection_phase("stopping", "检测已停止，停止创建新线程") final_phase = "stopped" final_detail = "检测已停止，停止创建新线程" break pending_entry = pending_domains.popleft() domain = pending_entry.get("domain") or {} using_job_queue = bool(pending_entry.get("using_job_queue")) if start_domain_thread(domain, using_job_queue, max_threads): dispatched_this_round += 1 # 派发大批量任务时，尽早把刚启动的任务项从 claimed 刷到 running， # 避免数据库状态长期滞后，影响 controller 对真实活跃量的判断。 if dispatched_this_round % 64 == 0: self._flush_pending_running_marks() if dispatched_this_round: dispatch_summary_message = ( f"持续补位派发完成: dispatched={dispatched_this_round} | " f"active_threads={len(inflight_futures)} | pending_buffer={len(pending_domains)} | " f"max_threads={max_threads} | queued_backlog={queued_backlog} | backlog_limit={backlog_limit}" ) logger.info(dispatch_summary_message) self._sync_worker_log_event( dispatch_summary_message, payload={ "dispatched": dispatched_this_round, "active_threads": len(inflight_futures), "pending_buffer": len(pending_domains), "max_threads": max_threads, "queued_backlog": queued_backlog, "backlog_limit": backlog_limit, }, mode='full', ) if not pending_domains and not inflight_futures: if legacy_tail_reached: logger.info("兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成") self._mark_detection_phase( "completing", "兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成", processed=total_processed, ) self._sync_worker_log_event( "兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成", payload={"processed": total_processed}, mode='key', ) break logger.info("没有需要检测的域名") self._mark_detection_phase("idle", "当前没有需要检测的域名，Worker 等待下一次启动") self._sync_worker_log_event("当前没有需要检测的域名，Worker 等待下一次启动", mode='key') break emit_thread_progress(max_threads, batch_size=len(pending_domains) + len(inflight_futures)) time.sleep(0.05 if pending_domains or inflight_futures else 0.2) finally: self._flush_pending_running_marks(force=True, batch_limit=5000) self._flush_pending_job_releases(force=True, batch_limit=5000) self._flush_pending_job_finalizations(force=True, batch_limit=5000) self._flush_pending_domain_completions(force=True, batch_limit=5000) self._flush_pending_domain_status_updates(force=True, batch_limit=5000) self._flush_pending_review_status_updates(force=True, batch_limit=5000) if self.current_job_id: self.db.refresh_detect_job_status(self.current_job_id) can_blocking_shutdown = ( not pending_domains and not inflight_futures and self._get_active_domain_threads() <= 0 ) worker_pool.shutdown(wait=can_blocking_shutdown, cancel_futures=True) if can_blocking_shutdown: gc.collect() # 完成进度 if self.detect_thread: try: # 通过信号发送进度更新 self.detect_thread.progress_signal.emit(100, 100) except Exception as e: logger.error(f"发送进度更新信号失败: {e}") import traceback logger.error(traceback.format_exc()) logger.info("域名检测任务完成") self._mark_detection_phase("completed", "域名检测任务完成") self._sync_worker_log_event("域名检测任务完成", payload={"processed": total_processed}, mode='key') final_phase = "completed" final_detail = "域名检测任务完成" except Exception as e: logger.error(f"执行检测任务出错: {e}") self._mark_detection_phase("failed", f"执行检测任务出错: {e}") self._sync_worker_log_event(f"执行检测任务出错: {e}", level='error', mode='key') final_phase = "failed" final_detail = f"执行检测任务出错: {e}" finally: self.detecting = False if not self.running or self.stop_requested: final_phase = "stopped" final_detail = "检测任务已停止" self._update_runtime_state( final_phase, final_detail, active_threads=0, max_threads=max(1, int(getattr(self, "thread_count", 1) or 1)), processed=total_processed, ) def run_daily_task(self): """ 执行每日检测任务 """ logger.info(f"[{datetime.now()}] 开始每日检测任务") self.start_detection() logger.info(f"[{datetime.now()}] 每日检测任务完成") def start_scheduler(self): """ 启动定时任务 """ # 避免重复注册相同定时任务 schedule.clear('daily_detection') # 每天凌晨1点执行检测；启动程序时不自动跑一轮，避免和人工启动混淆 schedule.every().day.at("01:00").do(self.run_daily_task).tag('daily_detection') logger.info("定时任务已启动，每天凌晨1点执行检测；程序启动后不会立即自动执行") # 循环执行定时任务 while self.running: schedule.run_pending() time.sleep(60) def start(self): """ 启动检测端 """ logger.info("启动域名检测端") self.running = True # 启动定时任务 scheduler_thread = threading.Thread(target=self.start_scheduler) scheduler_thread.daemon = True scheduler_thread.start() # 保持程序运行 try: while self.running: time.sleep(1) except KeyboardInterrupt: logger.info("检测端已停止") self.running = False def load_cookies_from_remote(self): """ 从远程加载cookies """ try: if self.use_redis: # 加载聚名cookies juming_cookies_str = self.redis_client.get('domain_tool:juming_cookies') if juming_cookies_str: # 将cookies保存到本地文件 import pickle from requests.cookies import RequestsCookieJar # 创建cookie jar cookie_jar = RequestsCookieJar() # 解析cookies字符串 try: import ast cookies_dict = ast.literal_eval(juming_cookies_str) for name, value in cookies_dict.items(): cookie_jar.set(name, value) except Exception as e: logger.error(f"解析聚名cookies失败: {e}") # 保存到本地文件 try: with open('juming_cookies.pkl', 'wb') as f: pickle.dump(cookie_jar, f) logger.info("从Redis加载聚名cookies成功并保存到本地") except Exception as e: logger.error(f"保存聚名cookies到本地失败: {e}") # 加载聚查cookies jucha_cookies_str = self.redis_client.get('domain_tool:jucha_cookies') if jucha_cookies_str: # 将cookies保存到本地文件 import pickle from requests.cookies import RequestsCookieJar # 创建cookie jar cookie_jar = RequestsCookieJar() # 解析cookies字符串 try: import ast cookies_dict = ast.literal_eval(jucha_cookies_str) for name, value in cookies_dict.items(): cookie_jar.set(name, value) except Exception as e: logger.error(f"解析聚查cookies失败: {e}") # 保存到本地文件 try: with open('jucha_cookies.pkl', 'wb') as f: pickle.dump(cookie_jar, f) logger.info("从Redis加载聚查cookies成功并保存到本地") except Exception as e: logger.error(f"保存聚查cookies到本地失败: {e}") # 加载桔子SEO cookies juziseo_cookies_str = self.redis_client.get('domain_tool:juziseo_cookies') if juziseo_cookies_str: # 将cookies保存到本地文件 import pickle from requests.cookies import RequestsCookieJar # 创建cookie jar cookie_jar = RequestsCookieJar() # 解析cookies字符串 try: import ast cookies_dict = ast.literal_eval(juziseo_cookies_str) for name, value in cookies_dict.items(): cookie_jar.set(name, value) except Exception as e: logger.error(f"解析桔子SEO cookies失败: {e}") # 保存到本地文件 try: with open('juziseo_cookies.pkl', 'wb') as f: pickle.dump(cookie_jar, f) logger.info("从Redis加载桔子SEO cookies成功并保存到本地") except Exception as e: logger.error(f"保存桔子SEO cookies到本地失败: {e}") except Exception as e: logger.error(f"从远程加载cookies失败: {e}") def update_config_labels(self): """ 更新GUI配置标签 """ # 使用QCoreApplication.postEvent发送配置更新事件，这是最安全的方法 try: logger.debug("发送配置更新事件") # 确保主窗口存在 if self.main_window: from PySide6.QtCore import QCoreApplication event = ConfigUpdateEvent() QCoreApplication.postEvent(self.main_window, event) logger.debug("配置更新事件已发送") else: logger.debug("主窗口不存在，跳过配置更新") except Exception as e: logger.error(f"发送配置更新事件失败: {e}") def start_redis_subscription(self): """ 启动Redis订阅，监听配置更新 """ while self.running: pubsub = None try: # 创建新的Redis客户端用于订阅 redis_sub_client = get_redis_client(role="pubsub") # 订阅配置更新频道 pubsub = redis_sub_client.pubsub() pubsub.subscribe(CONFIG_UPDATE_CHANNEL, CONTROL_CHANNEL) # 使用logger.debug输出到文件日志，不输出到GUI日志框 logger.debug("开始监听配置更新...") self._consume_pending_control_command() self._resume_active_detect_job_if_needed(reason="redis_subscription_bootstrap") # 循环监听消息 while self.running: message = pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0) if not message: continue if message['type'] == 'message': channel = message.get('channel', '') payload = message['data'] if channel == CONFIG_UPDATE_CHANNEL: config_type = payload logger.debug(f"收到配置更新消息: {config_type}") previous_proxy_config = dict(self.proxy_config or {}) self.detect_options = self.load_detect_options() self.proxy_config = self.load_proxy_config() previous_thread_count = int(getattr(self, "thread_count", 0) or 0) self.thread_count = self.load_thread_count() self.sensitive_words = self._load_sensitive_words_runtime() self.proxy_max_reuse_count = max(8, self.thread_count * 4) self.runtime_settings = self.load_runtime_settings() self.load_cookies_from_remote() self.update_config_labels() should_refresh_proxy, reset_proxy_cooldown = self._should_trigger_proxy_refresh_for_config_update( config_type=config_type, previous_proxy_config=previous_proxy_config, previous_thread_count=previous_thread_count, current_thread_count=self.thread_count, ) if should_refresh_proxy: self.trigger_proxy_refresh( reason=f"config_update:{config_type}", reset_cooldown=reset_proxy_cooldown, ) logger.debug("配置已更新") continue if channel == CONTROL_CHANNEL: logger.info(f"收到 Worker 控制消息: {payload}") self._handle_control_message(payload) except redis.exceptions.TimeoutError: # Pub/Sub 空闲超时在 Linux 服务态下属于常见现象，不需要中断订阅。 continue except Exception as e: logger.debug(f"Redis订阅失败: {e}") logger.debug("3秒后尝试重新连接...") time.sleep(3) # 增加重试间隔 finally: if pubsub is not None: try: pubsub.close() except Exception: pass def stop(self): """ 停止检测端 """ logger.info("停止域名检测端") self._runtime_heartbeat_stop.set() self._worker_log_sync_stop.set() try: self._worker_log_sync_queue.put_nowait(None) except Exception: pass self.stop_requested = True self.running = False self.detecting = False self._update_runtime_state("stopped", "Worker 已停止") if __name__ == "__main__": try: # 配置日志 log_file_path = _resolve_worker_log_file() logger.add(log_file_path, rotation="1 day", level="DEBUG") _raise_nofile_soft_limit() logger.info("程序开始运行") # 创建应用程序 app = QApplication([]) logger.info("创建应用程序成功") # 创建并显示主窗口 window = DetectMainWindow() logger.info("创建主窗口成功") window.show() logger.info("显示主窗口成功") # 运行应用程序 logger.info("开始运行应用程序") app.exec() logger.info("应用程序运行结束") # 当窗口关闭时，停止检测端 if window.worker: logger.info("停止检测端") window.worker.stop() except Exception as e: logger.error(f"程序运行出错: {e}") import traceback logger.error(traceback.format_exc()) import time time.sleep(10) # 等待10秒，以便查看错误信息