admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
domainCheck/app/detectors/aizhan_detector.py
View File

@@ -64,7 +64,7 @@ class AizhanDetector(BaseDetector):
}
else:
self._log_warning(f"爱站网查询失败: {response.status_code}")
return {'title': '', 'risk': '', 'has_sensitive': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
@@ -127,4 +127,4 @@ class AizhanDetector(BaseDetector):
if word in title:
return True
return False
return False

12
domainCheck/app/detectors/baidu_detector.py
View File

@@ -83,10 +83,9 @@ class BaiduDetector(BaseDetector):
}
else:
self._log_warning(f"百度site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
return self._handle_exception(e, domain)
def check_history(self, domain):
"""
@@ -123,10 +122,9 @@ class BaiduDetector(BaseDetector):
}
else:
self._log_warning(f"百度历史查询失败: {response.status_code}")
return {'has_history': False, 'has_gray': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_history': False, 'has_gray': False}
return self._handle_exception(e, domain)
def _extract_subdomains(self, content, domain):
"""
@@ -148,4 +146,4 @@ class BaiduDetector(BaseDetector):
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []
return []

4
domainCheck/app/detectors/chinaz_detector.py
View File

@@ -64,7 +64,7 @@ class ChinazDetector(BaseDetector):
}
else:
self._log_warning(f"站长之家查询失败: {response.status_code}")
return {'title': '', 'category': '', 'has_sensitive': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
@@ -127,4 +127,4 @@ class ChinazDetector(BaseDetector):
if word in title:
return True
return False
return False

5
domainCheck/app/detectors/google_detector.py
View File

@@ -74,7 +74,6 @@ class GoogleDetector(BaseDetector):
}
else:
self._log_warning(f"Google site查询失败: {response.status_code}")
return {'has_收录': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False}
return self._handle_exception(e, domain)

17
domainCheck/app/detectors/jucha_detector.py
View File

@@ -81,10 +81,9 @@ class JuchaDetector(BaseDetector):
return whois_info
else:
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
return {'status': ''}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'status': ''}
return self._handle_exception(e, domain)
def check_beian(self, domain):
"""
@@ -113,10 +112,9 @@ class JuchaDetector(BaseDetector):
return beian_info
else:
self._log_warning(f"聚查备案查询失败: {response.status_code}")
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
return self._handle_exception(e, domain)
def check_intercept(self, domain):
"""
@@ -147,10 +145,9 @@ class JuchaDetector(BaseDetector):
}
else:
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
return {'normal': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'normal': False}
return self._handle_exception(e, domain)
def _extract_whois_info(self, content):
"""
@@ -225,4 +222,4 @@ class JuchaDetector(BaseDetector):
return False
except Exception as e:
self._handle_exception(e, 'check_intercept_status')
return False
return False

12
domainCheck/app/detectors/juziseo_detector.py
View File

@@ -93,10 +93,9 @@ class JuziseoDetector(BaseDetector):
}
else:
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
return self._handle_exception(e, domain)
def check_backlink(self, domain):
"""
@@ -131,10 +130,9 @@ class JuziseoDetector(BaseDetector):
}
else:
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_subdomains': False}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_subdomains': False}
return self._handle_exception(e, domain)
def _extract_history_info(self, content):
"""
@@ -211,4 +209,4 @@ class JuziseoDetector(BaseDetector):
if word in content:
return True
return False
return False

7
domainCheck/app/detectors/qihu360_detector.py
View File

@@ -79,10 +79,9 @@ class Qihu360Detector(BaseDetector):
}
else:
self._log_warning(f"360 site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
return {'error': f'HTTP {response.status_code}'}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
return self._handle_exception(e, domain)
def _extract_subdomains(self, content, domain):
"""
@@ -104,4 +103,4 @@ class Qihu360Detector(BaseDetector):
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []
return []

4
domainCheck/app/detectors/rdap_detector.py
View File

@@ -73,7 +73,7 @@ class RDAPDetector(BaseDetector):
return 2 # 可注册
# 检查域名状态
statuses = result.get('status', [])
statuses = result.get('statuses', [])
if 'clientHold' in statuses:
return 7 # clientHold
elif 'serverHold' in statuses:
@@ -125,4 +125,4 @@ class RDAPDetector(BaseDetector):
elif event_action == 'last update':
result['last_update'] = event_date
return result
return result

142
domainCheck/app/detectors/wayback_detector.py
View File

@@ -17,13 +17,13 @@ import zlib
from base64 import b64decode, b64encode
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
import redis
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from app.config import config
from app.detectors.base import BaseDetector
from app.utils.redis_client import get_redis_client
class WaybackDetector(BaseDetector):
@@ -51,6 +51,7 @@ class WaybackDetector(BaseDetector):
def _build_session(self):
session = requests.Session()
session.trust_env = False
retry = Retry(
total=max(0, config.WAYBACK_RETRY_COUNT),
backoff_factor=0.5,
@@ -106,15 +107,7 @@ class WaybackDetector(BaseDetector):
def _build_redis_client(self):
try:
client = redis.Redis(
host=config.REDIS_HOST,
port=config.REDIS_PORT,
password=config.REDIS_PASSWORD,
db=config.REDIS_DB,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
client = get_redis_client(role="standard")
client.ping()
return client
except Exception:
@@ -489,6 +482,14 @@ class WaybackDetector(BaseDetector):
trimmed.append(item)
return trimmed
def _resolve_scan_record_fetch_limit(self):
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
# 扫描阶段最终只会保留最近的少量快照；如果每次都把整个 CDX 历史拉下来，
# 在快照特别多的域名上会白白浪费很多秒。这里改成“最近一小窗”，同时留出
# 重复 digest/标题的缓冲空间，避免把最新几条里重复记录全部裁没了。
recent_window = max(12, max_records * 6)
return -recent_window
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
sensitive_words = sensitive_words or config.load_sensitive_words()
request_errors = []
@@ -512,12 +513,17 @@ class WaybackDetector(BaseDetector):
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
'elapsed_seconds': 0.0,
}
transient_request_failures = 0
transient_request_failure_threshold = 2
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
latest_fetch_transient_failure = False
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
if self._is_transient_request_error(latest_fetch.get('error')):
self._trip_transient_backoff()
latest_fetch_transient_failure = True
transient_request_failures += 1
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
@@ -547,26 +553,6 @@ class WaybackDetector(BaseDetector):
transient_snapshot_failures = 0
transient_snapshot_failure_threshold = max(2, domain_concurrency)
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
checked_snapshot_count = 1
@@ -605,40 +591,55 @@ class WaybackDetector(BaseDetector):
if latest_error:
request_errors.append(f"latest_snapshot: {latest_error}")
if latest_error and self._is_transient_request_error(latest_error):
transient_snapshot_failures += 1
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
transient_request_failures += 1
cached_records = self._load_cached_records(domain)
if cached_records is not None:
records = cached_records
else:
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
self._trip_transient_backoff()
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
if latest_fetch_transient_failure and not latest_timestamp:
records = []
else:
records_fetch = self._fetch_cdx_records_with_meta(
domain,
limit=self._resolve_scan_record_fetch_limit(),
)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
transient_request_failures += 1
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
if (
not latest_timestamp
and not records
and (
transient_request_failures >= transient_request_failure_threshold
or latest_fetch_transient_failure
)
):
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
records = self._filter_records_recent_years(records, recent_years=recent_years)
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
@@ -660,12 +661,12 @@ class WaybackDetector(BaseDetector):
digest_seen.add(digest)
pending_records.append(item)
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
executor = ThreadPoolExecutor(max_workers=domain_concurrency)
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
try:
while (index < len(pending_records) or pending) and not stop_requested:
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
timestamp = pending_records[index]['timestamp']
@@ -694,8 +695,8 @@ class WaybackDetector(BaseDetector):
request_errors.append(f"snapshot:{timestamp}: {error_message}")
if error_message and self._is_transient_request_error(error_message):
transient_snapshot_failures += 1
self._trip_transient_backoff()
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
self._trip_transient_backoff()
stop_requested = True
continue
@@ -722,10 +723,13 @@ class WaybackDetector(BaseDetector):
)
if config.WAYBACK_REQUEST_DELAY > 0:
time.sleep(config.WAYBACK_REQUEST_DELAY)
finally:
if stop_requested:
for future in pending:
for future in list(pending.keys()):
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
else:
executor.shutdown(wait=True)
return {
'snapshot_years': years,