admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
domainCheck/app/core/detect_engine.py
View File

@@ -11,6 +11,7 @@
import time
from loguru import logger
from app.utils.database import Database
from app.utils.detection_results import normalize_detector_result
from app.detectors.rdap_detector import RDAPDetector
from app.detectors.wayback_detector import WaybackDetector
from app.detectors.baidu_detector import BaiduDetector
@@ -32,6 +33,10 @@ class DetectEngine:
"""
检测引擎
"""
OUTCOME_SUCCESS = "success"
OUTCOME_BLACKLISTED = "blacklisted"
OUTCOME_FAILED = "failed"
def __init__(self):
"""
@@ -49,6 +54,9 @@ class DetectEngine:
self.jucha_detector = JuchaDetector()
def detect_domain(self, domain_id):
return self._detect_domain_with_outcome(domain_id) == self.OUTCOME_SUCCESS
def _detect_domain_with_outcome(self, domain_id):
"""
检测域名
@@ -69,24 +77,30 @@ class DetectEngine:
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
# 1. 基础检测
if not self._basic_detect(domain_id, domain):
basic_outcome = self._basic_detect(domain_id, domain)
if basic_outcome != self.OUTCOME_SUCCESS:
logger.info(f"基础检测失败，停止后续检测: {domain}")
return False
if basic_outcome == self.OUTCOME_FAILED:
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
return basic_outcome
# 2. 深度检测
if not self._deep_detect(domain_id, domain):
deep_outcome = self._deep_detect(domain_id, domain)
if deep_outcome != self.OUTCOME_SUCCESS:
logger.info(f"深度检测失败: {domain}")
return False
if deep_outcome == self.OUTCOME_FAILED:
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
return deep_outcome
# 更新检测状态为正常
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED)
logger.info(f"域名检测完成: {domain}")
return True
return self.OUTCOME_SUCCESS
except Exception as e:
logger.error(f"检测域名出错: {e}")
# 更新检测状态为检测失败
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
return False
return self.OUTCOME_FAILED
def _basic_detect(self, domain_id, domain):
"""
@@ -94,7 +108,7 @@ class DetectEngine:
:param domain_id: 域名ID
:param domain: 域名
:return: bool - 是否检测通过
:return: str - 检测结果
"""
# 1. 检查是否为一口价域名
is_ykj = self.db.is_ykj_domain(domain_id)
@@ -108,7 +122,7 @@ class DetectEngine:
if self.db.is_blacklisted(domain):
logger.info(f"域名在黑名单中: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
return False
return self.OUTCOME_BLACKLISTED
# 4. 时光机快照年份采集
snapshot_years = self.wayback_detector.get_snapshot_years(domain)
@@ -120,9 +134,9 @@ class DetectEngine:
logger.info(f"域名包含敏感词: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "快照包含敏感词")
return False
return self.OUTCOME_BLACKLISTED
return True
return self.OUTCOME_SUCCESS
def _deep_detect(self, domain_id, domain):
"""
@@ -130,38 +144,52 @@ class DetectEngine:
:param domain_id: 域名ID
:param domain: 域名
:return: bool - 是否检测通过
:return: str - 检测结果
"""
# 1. 百度历史/Site
baidu_history = self.baidu_detector.check_history(domain)
baidu_site = self.baidu_detector.check_site(domain)
# 2. 360 Site
qihu360_site = self.qihu360_detector.check_site(domain)
# 3. Google Site
google_site = self.google_detector.check_site(domain)
# 4. 站长之家
chinaz_info = self.chinaz_detector.check_domain(domain)
# 5. 爱站网
aizhan_info = self.aizhan_detector.check_domain(domain)
# 6. 桔子SEO
juziseo_info = self.juziseo_detector.check_domain(domain)
# 7. 聚查
jucha_info = self.jucha_detector.check_domain(domain)
# 检查是否有风险
if self._check_risk(domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
return False
detector_results = {}
detector_steps = [
("baidu_history", lambda: self.baidu_detector.check_history(domain)),
("baidu_site", lambda: self.baidu_detector.check_site(domain)),
("qihu360_site", lambda: self.qihu360_detector.check_site(domain)),
("google_site", lambda: self.google_detector.check_site(domain)),
("chinaz_info", lambda: self.chinaz_detector.check_domain(domain)),
("aizhan_info", lambda: self.aizhan_detector.check_domain(domain)),
("juziseo_info", lambda: self.juziseo_detector.check_domain(domain)),
("jucha_info", lambda: self.jucha_detector.check_domain(domain)),
]
for detector_name, runner in detector_steps:
detector_results[detector_name] = normalize_detector_result(detector_name, runner())
detector_error = detector_results[detector_name].get("error")
if detector_error:
logger.error(
f"深度检测存在第三方检测错误: {domain}, detector={detector_name}, error={detector_error}"
)
return self.OUTCOME_FAILED
if self._check_risk(
domain_id,
domain,
detector_results.get("baidu_history"),
detector_results.get("baidu_site"),
detector_results.get("qihu360_site"),
detector_results.get("google_site"),
detector_results.get("chinaz_info"),
detector_results.get("aizhan_info"),
detector_results.get("juziseo_info"),
detector_results.get("jucha_info"),
):
self._persist_detection_results(domain_id, detector_results)
return self.OUTCOME_BLACKLISTED
# 保存检测结果
self.db.add_detection_result(domain_id, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info)
persisted = self._persist_detection_results(domain_id, detector_results)
if not persisted:
logger.error(f"保存检测结果失败: {domain}")
return self.OUTCOME_FAILED
return True
return self.OUTCOME_SUCCESS
def _check_risk(self, domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
"""
@@ -180,7 +208,9 @@ class DetectEngine:
:return: bool - 是否有风险
"""
# 检查百度历史过灰
if baidu_history and '灰' in str(baidu_history):
if (
isinstance(baidu_history, dict) and baidu_history.get('has_gray')
) or (baidu_history and '灰' in str(baidu_history)):
logger.info(f"百度历史过灰: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "百度历史过灰")
@@ -215,7 +245,7 @@ class DetectEngine:
# 检查WHOIS状态
if jucha_info and 'whois' in jucha_info:
if jucha_info['whois'].get('status') in ['clientHold', 'serverHold']:
if jucha_info['whois'].get('whois_status') in ['clientHold', 'serverHold']:
logger.info(f"WHOIS状态异常: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "WHOIS状态异常")
@@ -225,11 +255,24 @@ class DetectEngine:
if jucha_info and 'intercept' in jucha_info:
if not jucha_info['intercept'].get('normal', True):
logger.info(f"拦截检测异常: {domain}")
self.db.update_domain_detect_status(domain_id, 4) # 4 表示黑名单
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "拦截检测异常")
return True
return False
def _persist_detection_results(self, domain_id, detector_results):
return self.db.add_detection_result(
domain_id,
detector_results.get("baidu_history"),
detector_results.get("baidu_site"),
detector_results.get("qihu360_site"),
detector_results.get("google_site"),
detector_results.get("chinaz_info"),
detector_results.get("aizhan_info"),
detector_results.get("juziseo_info"),
detector_results.get("jucha_info"),
)
def _contains_sensitive_words(self, text):
"""
@@ -264,11 +307,12 @@ class DetectEngine:
self.db.update_task_status(task_id, 1) # 1 表示执行中
# 执行检测
success = self.detect_domain(domain_id)
outcome = self._detect_domain_with_outcome(domain_id)
# 更新任务状态
if success:
if outcome in (self.OUTCOME_SUCCESS, self.OUTCOME_BLACKLISTED):
self.db.update_task_status(task_id, 2) # 2 表示完成
return True
else:
# 增加重试次数
retry_count = task.get('retry_count', 0) + 1
@@ -277,8 +321,7 @@ class DetectEngine:
self.db.update_task_status(task_id, 0) # 0 表示待执行
else:
self.db.update_task_status(task_id, 3) # 3 表示失败
return success
return False
except Exception as e:
logger.error(f"处理任务出错: {e}")
# 更新任务状态为失败

4
domainCheck/app/detectors/aizhan_detector.py
View File

@@ -64,7 +64,7 @@ class AizhanDetector(BaseDetector):
}
else:
self._log_warning(f"爱站网查询失败: {response.status_code}")
return {'title': '', 'risk': '', 'has_sensitive': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
@@ -127,4 +127,4 @@ class AizhanDetector(BaseDetector):
if word in title:
return True
return False
return False

12
domainCheck/app/detectors/baidu_detector.py
View File

@@ -83,10 +83,9 @@ class BaiduDetector(BaseDetector):
}
else:
self._log_warning(f"百度site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
return self._handle_exception(e, domain)
def check_history(self, domain):
"""
@@ -123,10 +122,9 @@ class BaiduDetector(BaseDetector):
}
else:
self._log_warning(f"百度历史查询失败: {response.status_code}")
return {'has_history': False, 'has_gray': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_history': False, 'has_gray': False}
return self._handle_exception(e, domain)
def _extract_subdomains(self, content, domain):
"""
@@ -148,4 +146,4 @@ class BaiduDetector(BaseDetector):
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []
return []

4
domainCheck/app/detectors/chinaz_detector.py
View File

@@ -64,7 +64,7 @@ class ChinazDetector(BaseDetector):
}
else:
self._log_warning(f"站长之家查询失败: {response.status_code}")
return {'title': '', 'category': '', 'has_sensitive': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
@@ -127,4 +127,4 @@ class ChinazDetector(BaseDetector):
if word in title:
return True
return False
return False

5
domainCheck/app/detectors/google_detector.py
View File

@@ -74,7 +74,6 @@ class GoogleDetector(BaseDetector):
}
else:
self._log_warning(f"Google site查询失败: {response.status_code}")
return {'has_收录': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False}
return self._handle_exception(e, domain)

17
domainCheck/app/detectors/jucha_detector.py
View File

@@ -81,10 +81,9 @@ class JuchaDetector(BaseDetector):
return whois_info
else:
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
return {'status': ''}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'status': ''}
return self._handle_exception(e, domain)
def check_beian(self, domain):
"""
@@ -113,10 +112,9 @@ class JuchaDetector(BaseDetector):
return beian_info
else:
self._log_warning(f"聚查备案查询失败: {response.status_code}")
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
return self._handle_exception(e, domain)
def check_intercept(self, domain):
"""
@@ -147,10 +145,9 @@ class JuchaDetector(BaseDetector):
}
else:
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
return {'normal': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'normal': False}
return self._handle_exception(e, domain)
def _extract_whois_info(self, content):
"""
@@ -225,4 +222,4 @@ class JuchaDetector(BaseDetector):
return False
except Exception as e:
self._handle_exception(e, 'check_intercept_status')
return False
return False

12
domainCheck/app/detectors/juziseo_detector.py
View File

@@ -93,10 +93,9 @@ class JuziseoDetector(BaseDetector):
}
else:
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
return self._handle_exception(e, domain)
def check_backlink(self, domain):
"""
@@ -131,10 +130,9 @@ class JuziseoDetector(BaseDetector):
}
else:
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_subdomains': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_subdomains': False}
return self._handle_exception(e, domain)
def _extract_history_info(self, content):
"""
@@ -211,4 +209,4 @@ class JuziseoDetector(BaseDetector):
if word in content:
return True
return False
return False

7
domainCheck/app/detectors/qihu360_detector.py
View File

@@ -79,10 +79,9 @@ class Qihu360Detector(BaseDetector):
}
else:
self._log_warning(f"360 site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
return self._handle_exception(e, domain)
def _extract_subdomains(self, content, domain):
"""
@@ -104,4 +103,4 @@ class Qihu360Detector(BaseDetector):
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []
return []

4
domainCheck/app/detectors/rdap_detector.py
View File

@@ -73,7 +73,7 @@ class RDAPDetector(BaseDetector):
return 2 # 可注册
# 检查域名状态
statuses = result.get('status', [])
statuses = result.get('statuses', [])
if 'clientHold' in statuses:
return 7 # clientHold
elif 'serverHold' in statuses:
@@ -125,4 +125,4 @@ class RDAPDetector(BaseDetector):
elif event_action == 'last update':
result['last_update'] = event_date
return result
return result

142
domainCheck/app/detectors/wayback_detector.py
View File

@@ -17,13 +17,13 @@ import zlib
from base64 import b64decode, b64encode
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
import redis
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from app.config import config
from app.detectors.base import BaseDetector
from app.utils.redis_client import get_redis_client
class WaybackDetector(BaseDetector):
@@ -51,6 +51,7 @@ class WaybackDetector(BaseDetector):
def _build_session(self):
session = requests.Session()
session.trust_env = False
retry = Retry(
total=max(0, config.WAYBACK_RETRY_COUNT),
backoff_factor=0.5,
@@ -106,15 +107,7 @@ class WaybackDetector(BaseDetector):
def _build_redis_client(self):
try:
client = redis.Redis(
host=config.REDIS_HOST,
port=config.REDIS_PORT,
password=config.REDIS_PASSWORD,
db=config.REDIS_DB,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
client = get_redis_client(role="standard")
client.ping()
return client
except Exception:
@@ -489,6 +482,14 @@ class WaybackDetector(BaseDetector):
trimmed.append(item)
return trimmed
def _resolve_scan_record_fetch_limit(self):
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
# 扫描阶段最终只会保留最近的少量快照；如果每次都把整个 CDX 历史拉下来，
# 在快照特别多的域名上会白白浪费很多秒。这里改成“最近一小窗”，同时留出
# 重复 digest/标题的缓冲空间，避免把最新几条里重复记录全部裁没了。
recent_window = max(12, max_records * 6)
return -recent_window
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
sensitive_words = sensitive_words or config.load_sensitive_words()
request_errors = []
@@ -512,12 +513,17 @@ class WaybackDetector(BaseDetector):
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
'elapsed_seconds': 0.0,
}
transient_request_failures = 0
transient_request_failure_threshold = 2
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
latest_fetch_transient_failure = False
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
if self._is_transient_request_error(latest_fetch.get('error')):
self._trip_transient_backoff()
latest_fetch_transient_failure = True
transient_request_failures += 1
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
@@ -547,26 +553,6 @@ class WaybackDetector(BaseDetector):
transient_snapshot_failures = 0
transient_snapshot_failure_threshold = max(2, domain_concurrency)
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
checked_snapshot_count = 1
@@ -605,40 +591,55 @@ class WaybackDetector(BaseDetector):
if latest_error:
request_errors.append(f"latest_snapshot: {latest_error}")
if latest_error and self._is_transient_request_error(latest_error):
transient_snapshot_failures += 1
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
transient_request_failures += 1
cached_records = self._load_cached_records(domain)
if cached_records is not None:
records = cached_records
else:
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
self._trip_transient_backoff()
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
if latest_fetch_transient_failure and not latest_timestamp:
records = []
else:
records_fetch = self._fetch_cdx_records_with_meta(
domain,
limit=self._resolve_scan_record_fetch_limit(),
)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
transient_request_failures += 1
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
if (
not latest_timestamp
and not records
and (
transient_request_failures >= transient_request_failure_threshold
or latest_fetch_transient_failure
)
):
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
records = self._filter_records_recent_years(records, recent_years=recent_years)
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
@@ -660,12 +661,12 @@ class WaybackDetector(BaseDetector):
digest_seen.add(digest)
pending_records.append(item)
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
executor = ThreadPoolExecutor(max_workers=domain_concurrency)
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
try:
while (index < len(pending_records) or pending) and not stop_requested:
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
timestamp = pending_records[index]['timestamp']
@@ -694,8 +695,8 @@ class WaybackDetector(BaseDetector):
request_errors.append(f"snapshot:{timestamp}: {error_message}")
if error_message and self._is_transient_request_error(error_message):
transient_snapshot_failures += 1
self._trip_transient_backoff()
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
self._trip_transient_backoff()
stop_requested = True
continue
@@ -722,10 +723,13 @@ class WaybackDetector(BaseDetector):
)
if config.WAYBACK_REQUEST_DELAY > 0:
time.sleep(config.WAYBACK_REQUEST_DELAY)
finally:
if stop_requested:
for future in pending:
for future in list(pending.keys()):
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
else:
executor.shutdown(wait=True)
return {
'snapshot_years': years,

75
domainCheck/app/ui/domain_filter.py
View File

@@ -8,6 +8,8 @@
@explain : 域名筛选界面
'''
import json
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QPushButton, QLabel, QComboBox, QDateEdit, QCheckBox, QTableWidget, QTableWidgetItem, QHeaderView, QFileDialog, QLineEdit, QSpinBox, QInputDialog
from PySide6.QtGui import QIntValidator
from PySide6.QtCore import Qt, QDate, QThread, Signal
@@ -15,6 +17,11 @@ from loguru import logger
from app.core.export_manager import ExportManager
from app.utils.database import Database
from app.utils.detection_results import (
build_manual_detection_result,
load_detection_result,
resolve_detection_status,
)
from app.utils.status_codes import (
DETECT_STATUS_BLACKLISTED,
DETECT_STATUS_COMPLETED,
@@ -205,9 +212,9 @@ class UpdateThread(QThread):
baidu_history_value = self.update_values['baidu_history']
if baidu_history_value != '不更新':
status_value = status_mappings['百度历史收录状态'][baidu_history_value]
# 转换为JSON格式
import json
json_value = json.dumps({"status": status_value})
json_value = json.dumps(
build_manual_detection_result(status_value, legacy_key='has_history')
)
if detection_id:
cur.execute("UPDATE domain_detections SET baidu_history = %s WHERE domain_id = %s", (json_value, domain_info['id']))
else:
@@ -219,9 +226,9 @@ class UpdateThread(QThread):
baidu_site_value = self.update_values['baidu_site']
if baidu_site_value != '不更新':
status_value = status_mappings['百度site收录状态'][baidu_site_value]
# 转换为JSON格式
import json
json_value = json.dumps({"status": status_value})
json_value = json.dumps(
build_manual_detection_result(status_value, legacy_key='has_收录')
)
if detection_id:
cur.execute("UPDATE domain_detections SET baidu_site = %s WHERE domain_id = %s", (json_value, domain_info['id']))
else:
@@ -244,9 +251,9 @@ class UpdateThread(QThread):
qihu360_site_value = self.update_values['qihu360_site']
if qihu360_site_value != '不更新':
status_value = status_mappings['360 site收录状态'][qihu360_site_value]
# 转换为JSON格式
import json
json_value = json.dumps({"status": status_value})
json_value = json.dumps(
build_manual_detection_result(status_value, legacy_key='has_收录')
)
if detection_id:
cur.execute("UPDATE domain_detections SET qihu360_site = %s WHERE domain_id = %s", (json_value, domain_info['id']))
else:
@@ -258,9 +265,9 @@ class UpdateThread(QThread):
google_site_value = self.update_values['google_site']
if google_site_value != '不更新':
status_value = status_mappings['Google site收录状态'][google_site_value]
# 转换为JSON格式
import json
json_value = json.dumps({"status": status_value})
json_value = json.dumps(
build_manual_detection_result(status_value, legacy_key='has_收录')
)
if detection_id:
cur.execute("UPDATE domain_detections SET google_site = %s WHERE domain_id = %s", (json_value, domain_info['id']))
else:
@@ -1082,28 +1089,14 @@ class DomainFilterWidget(QWidget):
# 百度历史收录状态
baidu_history = domain.get('baidu_history')
if baidu_history is None:
baidu_history = {}
elif isinstance(baidu_history, str):
import json
try:
baidu_history = json.loads(baidu_history)
except:
baidu_history = {}
baidu_history_status = '是' if baidu_history.get('status') else '否'
baidu_history = load_detection_result(baidu_history)
baidu_history_status = '是' if resolve_detection_status(baidu_history, 'has_history') else '否'
self.table_widget.setItem(row, 12, QTableWidgetItem(baidu_history_status))
# 百度site收录状态
baidu_site = domain.get('baidu_site')
if baidu_site is None:
baidu_site = {}
elif isinstance(baidu_site, str):
import json
try:
baidu_site = json.loads(baidu_site)
except:
baidu_site = {}
baidu_site_status = '是' if baidu_site.get('status') else '否'
baidu_site = load_detection_result(baidu_site)
baidu_site_status = '是' if resolve_detection_status(baidu_site, 'has_收录') else '否'
self.table_widget.setItem(row, 13, QTableWidgetItem(baidu_site_status))
# title是否有中文
@@ -1113,28 +1106,14 @@ class DomainFilterWidget(QWidget):
# 360site收录
qihu360_site = domain.get('qihu360_site')
if qihu360_site is None:
qihu360_site = {}
elif isinstance(qihu360_site, str):
import json
try:
qihu360_site = json.loads(qihu360_site)
except:
qihu360_site = {}
qihu360_site_status = '是' if qihu360_site.get('status') else '否'
qihu360_site = load_detection_result(qihu360_site)
qihu360_site_status = '是' if resolve_detection_status(qihu360_site, 'has_收录') else '否'
self.table_widget.setItem(row, 15, QTableWidgetItem(qihu360_site_status))
# Google site收录状态
google_site = domain.get('google_site')
if google_site is None:
google_site = {}
elif isinstance(google_site, str):
import json
try:
google_site = json.loads(google_site)
except:
google_site = {}
google_site_status = '是' if google_site.get('status') else '否'
google_site = load_detection_result(google_site)
google_site_status = '是' if resolve_detection_status(google_site, 'has_收录') else '否'
self.table_widget.setItem(row, 16, QTableWidgetItem(google_site_status))
# 友情链接数量

1913
domainCheck/app/utils/database.py
View File

File diff suppressed because it is too large Load Diff

194
domainCheck/app/utils/detection_results.py Normal file
View File

@@ -0,0 +1,194 @@
import json
def build_detection_result(
*,
status=None,
state=None,
message="",
error=None,
**payload,
):
result = dict(payload)
result["status"] = bool(status) if status is not None else None
result["message"] = message or ""
if error:
result["state"] = "error"
result["error"] = str(error)
if not result["message"]:
result["message"] = str(error)
else:
result["state"] = state or _default_state_for_status(status)
result.pop("error", None)
return result
def _default_state_for_status(status):
if status is True:
return "positive"
if status is False:
return "negative"
return "ok"
def load_detection_result(value):
if value is None:
return {}
if isinstance(value, dict):
return value
if isinstance(value, str):
try:
decoded = json.loads(value)
except Exception:
return {}
return decoded if isinstance(decoded, dict) else {}
return {}
def resolve_detection_status(value, *legacy_keys):
data = load_detection_result(value)
if not data:
return False
status = data.get("status")
if status is not None:
return bool(status)
for key in legacy_keys:
if data.get(key) is not None:
return bool(data.get(key))
return False
def build_manual_detection_result(status, *, legacy_key=None, message="人工更新"):
payload = {}
if legacy_key:
payload[legacy_key] = bool(status)
return build_detection_result(
status=bool(status),
state="manual",
message=message,
**payload,
)
def normalize_detector_result(name, result):
result = load_detection_result(result)
if result.get("error"):
return build_detection_result(error=result.get("error"), **_without_meta(result))
if name == "baidu_history":
has_history = bool(result.get("has_history"))
has_gray = bool(result.get("has_gray"))
return build_detection_result(
status=has_history,
state="risk" if has_gray else None,
has_history=has_history,
has_gray=has_gray,
)
if name in {"baidu_site", "qihu360_site", "google_site"}:
has_index = bool(result.get("has_收录"))
normalized = build_detection_result(
status=has_index,
has_收录=has_index,
subdomains=list(result.get("subdomains", []) or []),
)
return normalized
if name == "chinaz_info":
return build_detection_result(
status=None,
title=result.get("title", ""),
category=result.get("category", ""),
has_sensitive=bool(result.get("has_sensitive")),
)
if name == "aizhan_info":
return build_detection_result(
status=None,
title=result.get("title", ""),
risk=result.get("risk", ""),
has_sensitive=bool(result.get("has_sensitive")),
)
if name == "juziseo_info":
history = normalize_detector_result("juziseo_history", result.get("history"))
backlink = normalize_detector_result("juziseo_backlink", result.get("backlink"))
nested_error = history.get("error") or backlink.get("error")
return build_detection_result(
status=None,
error=nested_error,
history=history,
backlink=backlink,
)
if name == "juziseo_history":
return build_detection_result(
status=None,
state="risk" if result.get("has_sensitive") or result.get("has_subdomains") else None,
has_sensitive=bool(result.get("has_sensitive")),
has_baidu_history=bool(result.get("has_baidu_history")),
has_subdomains=bool(result.get("has_subdomains")),
is_simplified=bool(result.get("is_simplified", True)),
)
if name == "juziseo_backlink":
return build_detection_result(
status=None,
state="risk" if result.get("has_sensitive") or result.get("has_subdomains") else None,
has_sensitive=bool(result.get("has_sensitive")),
has_subdomains=bool(result.get("has_subdomains")),
)
if name == "jucha_info":
whois = normalize_detector_result("jucha_whois", result.get("whois"))
beian = normalize_detector_result("jucha_beian", result.get("beian"))
intercept = normalize_detector_result("jucha_intercept", result.get("intercept"))
nested_error = whois.get("error") or beian.get("error") or intercept.get("error")
return build_detection_result(
status=None,
error=nested_error,
whois=whois,
beian=beian,
intercept=intercept,
)
if name == "jucha_whois":
whois_status = result.get("status", "")
hold = whois_status in {"clientHold", "serverHold"}
return build_detection_result(
status=None,
state="risk" if hold else None,
whois_status=whois_status,
)
if name == "jucha_beian":
has_beian = bool(result.get("has_beian"))
return build_detection_result(
status=has_beian,
has_beian=has_beian,
beian_year=result.get("beian_year", ""),
is_enterprise=bool(result.get("is_enterprise")),
beian_match=bool(result.get("beian_match")),
)
if name == "jucha_intercept":
normal = bool(result.get("normal"))
return build_detection_result(
status=normal,
state="risk" if not normal else None,
normal=normal,
)
return build_detection_result(status=None, **result)
def _without_meta(result):
return {
key: value
for key, value in result.items()
if key not in {"status", "state", "message", "error"}
}

129
domainCheck/app/utils/redis_client.py Normal file
View File

@@ -0,0 +1,129 @@
from __future__ import annotations
import os
import threading
import redis
from app.config import config
_CLIENTS: dict[tuple[str, bool], redis.Redis] = {}
_LOCK = threading.Lock()
def _safe_int(raw_value: object, default: int, minimum: int) -> int:
try:
parsed = int(raw_value)
except Exception:
parsed = default
return max(minimum, parsed)
def _safe_float(raw_value: object, default: float, minimum: float) -> float:
try:
parsed = float(raw_value)
except Exception:
parsed = default
return max(minimum, parsed)
def _pool_options(role: str, *, decode_responses: bool) -> dict:
normalized_role = str(role or "standard").strip().lower() or "standard"
node_code = str(getattr(config, "NODE_CODE", "") or "").strip() or "unknown"
if normalized_role == "pubsub":
return {
"host": config.REDIS_HOST,
"port": config.REDIS_PORT,
"password": config.REDIS_PASSWORD or None,
"db": config.REDIS_DB,
"decode_responses": decode_responses,
"socket_connect_timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_PUBSUB_CONNECT_TIMEOUT", "30"),
30.0,
1.0,
),
"socket_timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_PUBSUB_SOCKET_TIMEOUT", "60"),
60.0,
1.0,
),
"health_check_interval": _safe_int(
os.getenv("DOMAINCHECK_REDIS_HEALTH_CHECK_INTERVAL", "30"),
30,
0,
),
"retry_on_timeout": True,
"max_connections": _safe_int(
os.getenv("DOMAINCHECK_REDIS_PUBSUB_MAX_CONNECTIONS", "2"),
2,
1,
),
"timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_PUBSUB_POOL_TIMEOUT", "5"),
5.0,
0.1,
),
"client_name": f"domaincheck:pubsub:{node_code}:{os.getpid()}",
}
return {
"host": config.REDIS_HOST,
"port": config.REDIS_PORT,
"password": config.REDIS_PASSWORD or None,
"db": config.REDIS_DB,
"decode_responses": decode_responses,
"socket_connect_timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_CONNECT_TIMEOUT", "5"),
5.0,
0.5,
),
"socket_timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_SOCKET_TIMEOUT", "10"),
10.0,
0.5,
),
"health_check_interval": _safe_int(
os.getenv("DOMAINCHECK_REDIS_HEALTH_CHECK_INTERVAL", "30"),
30,
0,
),
"retry_on_timeout": True,
"max_connections": _safe_int(
os.getenv("DOMAINCHECK_REDIS_MAX_CONNECTIONS", "12"),
12,
1,
),
"timeout": _safe_float(
os.getenv("DOMAINCHECK_REDIS_POOL_TIMEOUT", "1.5"),
1.5,
0.1,
),
"client_name": f"domaincheck:standard:{node_code}:{os.getpid()}",
}
def get_redis_client(*, role: str = "standard", decode_responses: bool = True) -> redis.Redis:
normalized_role = str(role or "standard").strip().lower() or "standard"
cache_key = (normalized_role, bool(decode_responses))
with _LOCK:
cached = _CLIENTS.get(cache_key)
if cached is not None:
return cached
pool = redis.BlockingConnectionPool(**_pool_options(normalized_role, decode_responses=decode_responses))
client = redis.Redis(connection_pool=pool)
_CLIENTS[cache_key] = client
return client
def reset_redis_clients_for_tests() -> None:
with _LOCK:
clients = list(_CLIENTS.values())
_CLIENTS.clear()
for client in clients:
try:
client.close()
except Exception:
pass