admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first

Browse Source
This commit is contained in:
Your Name
2026-04-16 21:35:47 +08:00
parent ff32aa50bf
commit ebf632e651
86 changed files with 14097 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
domainCheck/app/config.py
View File

@@ -69,6 +69,9 @@ class Config:
REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
REDIS_DB = int(os.getenv('REDIS_DB', 0))
NODE_CODE = os.getenv('NODE_CODE', 'mainland-worker-01')
NODE_REGION = os.getenv('NODE_REGION', 'mainland')
NODE_ROLE = os.getenv('NODE_ROLE', 'worker')
# 聚名网配置
JUMING_COOKIE = os.getenv('JUMING_COOKIE', '')

45
domainCheck/app/detectors/wayback_detector.py
View File

@@ -221,7 +221,7 @@ class WaybackDetector(BaseDetector):
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
def _fetch_cdx_records_with_meta(self, domain, limit=None, fast_latest=False):
response = None
try:
params = {
@@ -242,7 +242,11 @@ class WaybackDetector(BaseDetector):
)
if response.status_code != 200:
self._log_warning(f"获取快照记录失败: {response.status_code}")
return []
return {
'records': [],
'error': f"HTTP {response.status_code}",
'status_code': response.status_code,
}
records = []
seen = set()
for raw_line in response.iter_lines(decode_unicode=True):
@@ -256,10 +260,18 @@ class WaybackDetector(BaseDetector):
continue
seen.add(timestamp)
records.append({'timestamp': timestamp, 'digest': digest})
return records
return {
'records': records,
'error': None,
'status_code': response.status_code,
}
except Exception as e:
self._handle_exception(e, domain)
return []
return {
'records': [],
'error': str(e),
'status_code': None,
}
finally:
try:
if response is not None:
@@ -267,6 +279,9 @@ class WaybackDetector(BaseDetector):
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
return self._fetch_cdx_records_with_meta(domain, limit=limit, fast_latest=fast_latest).get('records', [])
def get_latest_snapshot_record(self, domain):
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
return records[0] if records else None
@@ -391,7 +406,11 @@ class WaybackDetector(BaseDetector):
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
sensitive_words = sensitive_words or config.load_sensitive_words()
latest_record = self.get_latest_snapshot_record(domain)
request_errors = []
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
matched_word = None
@@ -444,7 +463,19 @@ class WaybackDetector(BaseDetector):
else:
failed_snapshot_count += 1
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
cached_records = self._load_cached_records(domain)
if cached_records is not None:
records = cached_records
else:
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
@@ -534,6 +565,8 @@ class WaybackDetector(BaseDetector):
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}

492
domainCheck/app/utils/database.py
View File

@@ -9,9 +9,11 @@
'''
import psycopg2
from psycopg2.extras import Json
import json
import redis
import threading
import socket
from loguru import logger
from app.config import config
from app.utils.status_codes import (
@@ -19,6 +21,7 @@ from app.utils.status_codes import (
DETECT_STATUS_COMPLETED,
DETECT_STATUS_FAILED,
DETECT_STATUS_PENDING,
DETECT_STATUS_RUNNING,
REGISTER_STATUS_AVAILABLE,
REGISTER_STATUS_REGISTERED,
REVIEW_STATUS_PENDING,
@@ -114,7 +117,7 @@ class Database:
self.use_bloom_filter = True
else:
# 如果命令不存在，说明 Redis 没有加载布隆过滤器模块
logger.warning(f"Redis 布隆过滤器不可用: {e}，将使用普通缓存")
logger.info(f"Redis 布隆过滤器不可用: {e}，将使用普通缓存")
self.use_bloom_filter = False
except Exception as e:
logger.warning(f"初始化布隆过滤器失败: {e}")
@@ -367,8 +370,464 @@ class Database:
pass
return False
finally:
# 将连接放回连接池
self.close(conn, cur)
def ensure_cluster_runtime_tables(self):
sql = """
CREATE TABLE IF NOT EXISTS detect_worker_nodes (
node_code VARCHAR(64) PRIMARY KEY,
region VARCHAR(32) NOT NULL DEFAULT 'unknown',
role VARCHAR(32) NOT NULL DEFAULT 'worker',
hostname VARCHAR(255) NOT NULL DEFAULT '',
ip VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'unknown',
worker_version VARCHAR(32) NOT NULL DEFAULT '',
current_load INTEGER NOT NULL DEFAULT 0,
metadata_json JSONB,
last_heartbeat_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS detect_jobs (
id BIGSERIAL PRIMARY KEY,
job_code VARCHAR(64) NOT NULL UNIQUE,
source VARCHAR(64) NOT NULL DEFAULT 'manual',
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
remark TEXT NOT NULL DEFAULT '',
created_by VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
started_at TIMESTAMP,
finished_at TIMESTAMP
);
CREATE TABLE IF NOT EXISTS detect_job_items (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
domain_id BIGINT NOT NULL,
status VARCHAR(32) NOT NULL DEFAULT 'pending',
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
claim_token VARCHAR(64) NOT NULL DEFAULT '',
lease_expires_at TIMESTAMP,
attempt_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT NOT NULL DEFAULT '',
result_version VARCHAR(64) NOT NULL DEFAULT '',
started_at TIMESTAMP,
finished_at TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
);
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
ON detect_job_items(status, lease_expires_at);
CREATE TABLE IF NOT EXISTS detect_run_events (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
job_item_id BIGINT REFERENCES detect_job_items(id) ON DELETE SET NULL,
node_code VARCHAR(64) NOT NULL DEFAULT '',
event_type VARCHAR(64) NOT NULL DEFAULT '',
level VARCHAR(16) NOT NULL DEFAULT 'info',
message TEXT NOT NULL DEFAULT '',
payload_json JSONB,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS detect_sync_records (
id BIGSERIAL PRIMARY KEY,
sync_type VARCHAR(32) NOT NULL DEFAULT '',
source_region VARCHAR(32) NOT NULL DEFAULT '',
target_region VARCHAR(32) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
payload_json JSONB,
error_message TEXT NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
"""
return self.execute(sql)
def register_cluster_node(self, node_code, region, role, status='online', current_load=0, metadata=None):
try:
ip_addr = ''
try:
ip_addr = socket.gethostbyname(socket.gethostname())
except Exception:
pass
sql = """
INSERT INTO detect_worker_nodes (
node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (node_code) DO UPDATE SET
region = EXCLUDED.region,
role = EXCLUDED.role,
hostname = EXCLUDED.hostname,
ip = EXCLUDED.ip,
status = EXCLUDED.status,
worker_version = EXCLUDED.worker_version,
current_load = EXCLUDED.current_load,
metadata_json = EXCLUDED.metadata_json,
last_heartbeat_at = CURRENT_TIMESTAMP,
update_time = CURRENT_TIMESTAMP
"""
return self.execute(
sql,
(
node_code,
region,
role,
socket.gethostname(),
ip_addr,
status,
'0.1.0',
int(current_load or 0),
json.dumps(metadata or {}, ensure_ascii=False),
),
)
except Exception as e:
logger.warning(f"注册检测节点失败: {e}")
return False
def claim_detect_job_items(self, node_code, limit=1000, lease_seconds=3600):
"""
领取一批待执行的任务项。
"""
conn = None
cur = None
claim_token = f"{node_code}-{int(threading.current_thread().ident or 0)}-{int(__import__('time').time())}"
try:
conn, cur = self.connect()
if not conn or not cur:
logger.error("领取检测任务失败: 无法获取数据库连接")
return []
cur.execute(
"""
WITH picked AS (
SELECT id
FROM detect_job_items
WHERE status IN ('pending', 'failed')
AND (lease_expires_at IS NULL OR lease_expires_at < CURRENT_TIMESTAMP)
ORDER BY create_time ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
),
updated AS (
UPDATE detect_job_items AS item
SET status = 'claimed',
claimed_by = %s,
claim_token = %s,
lease_expires_at = CURRENT_TIMESTAMP + (%s || ' seconds')::interval,
attempt_count = item.attempt_count + 1,
started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP),
updated_at = CURRENT_TIMESTAMP
FROM picked
WHERE item.id = picked.id
RETURNING item.id, item.job_id, item.domain_id, item.claim_token
)
SELECT
updated.id,
updated.job_id,
updated.domain_id,
updated.claim_token,
domains.domain,
domains.source_type,
domains.register_status,
domains.detect_status,
domains.use_status,
domains.expire_date,
domains.jucha_status,
domains.juziseo_status
FROM updated
JOIN domains ON domains.id = updated.domain_id
ORDER BY updated.id ASC
""",
(limit, node_code, claim_token, max(60, int(lease_seconds or 3600))),
)
rows = cur.fetchall() or []
touched_job_ids = sorted({row[1] for row in rows})
for job_id in touched_job_ids:
cur.execute(
"""
UPDATE detect_jobs
SET status = CASE WHEN status = 'pending' THEN 'running' ELSE status END,
started_at = COALESCE(started_at, CURRENT_TIMESTAMP)
WHERE id = %s
""",
(job_id,),
)
conn.commit()
return [
{
"job_item_id": row[0],
"job_id": row[1],
"id": row[2],
"claim_token": row[3],
"domain": row[4],
"source_type": row[5],
"register_status": row[6],
"detect_status": row[7],
"use_status": row[8],
"expire_date": row[9],
"jucha_status": row[10],
"juziseo_status": row[11],
}
for row in rows
]
except Exception as e:
try:
if conn:
conn.rollback()
except Exception:
pass
logger.error(f"领取检测任务失败: {e}")
return []
finally:
self.close(conn, cur)
def recycle_expired_detect_job_items(self):
"""
回收租约过期但未完成的任务项，重新放回 pending。
"""
conn = None
cur = None
try:
conn, cur = self.connect()
if not conn or not cur:
return 0
cur.execute(
"""
WITH recycled AS (
UPDATE detect_job_items
SET status = 'pending',
claimed_by = '',
claim_token = '',
lease_expires_at = NULL,
updated_at = CURRENT_TIMESTAMP,
last_error = CASE
WHEN status = 'running' THEN 'lease expired while running'
WHEN status = 'claimed' THEN 'lease expired before running'
ELSE last_error
END
WHERE status IN ('claimed', 'running')
AND lease_expires_at IS NOT NULL
AND lease_expires_at < CURRENT_TIMESTAMP
RETURNING job_id
)
SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL)
FROM recycled
"""
)
row = cur.fetchone()
recycled_count = int((row or [0])[0] or 0)
touched_job_ids = (row or [0, []])[1] or []
for job_id in touched_job_ids:
self._refresh_detect_job_status_with_cursor(cur, job_id)
conn.commit()
return recycled_count
except Exception as e:
try:
if conn:
conn.rollback()
except Exception:
pass
logger.error(f"回收过期检测任务失败: {e}")
return 0
finally:
self.close(conn, cur)
def release_detect_job_items_for_node(self, node_code):
"""
释放指定节点遗留的 claimed/running 任务项，供节点重启后快速恢复。
"""
conn = None
cur = None
try:
conn, cur = self.connect()
if not conn or not cur:
return 0
cur.execute(
"""
WITH released AS (
UPDATE detect_job_items
SET status = 'pending',
claimed_by = '',
claim_token = '',
lease_expires_at = NULL,
updated_at = CURRENT_TIMESTAMP,
last_error = CASE
WHEN status = 'running' THEN 'released after worker restart'
WHEN status = 'claimed' THEN 'released before execution after worker restart'
ELSE last_error
END
WHERE claimed_by = %s
AND status IN ('claimed', 'running')
RETURNING job_id
)
SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL)
FROM released
""",
(node_code,),
)
row = cur.fetchone()
released_count = int((row or [0])[0] or 0)
touched_job_ids = (row or [0, []])[1] or []
for job_id in touched_job_ids:
self._refresh_detect_job_status_with_cursor(cur, job_id)
conn.commit()
return released_count
except Exception as e:
try:
if conn:
conn.rollback()
except Exception:
pass
logger.error(f"释放节点遗留任务失败: {e}")
return 0
finally:
self.close(conn, cur)
def mark_detect_job_item_running(self, job_item_id, claim_token):
return self.execute(
"""
UPDATE detect_job_items
SET status = 'running',
updated_at = CURRENT_TIMESTAMP,
lease_expires_at = CURRENT_TIMESTAMP + interval '1 hour'
WHERE id = %s AND claim_token = %s
""",
(job_item_id, claim_token),
)
def renew_detect_job_item_lease(self, job_item_id, claim_token, lease_seconds=3600):
return self.execute(
"""
UPDATE detect_job_items
SET lease_expires_at = CURRENT_TIMESTAMP + (%s || ' seconds')::interval,
updated_at = CURRENT_TIMESTAMP
WHERE id = %s AND claim_token = %s AND status IN ('claimed', 'running')
""",
(max(60, int(lease_seconds or 3600)), job_item_id, claim_token),
)
def append_detect_run_event(self, job_id, job_item_id, node_code, event_type, message, level='info', payload=None):
return self.execute(
"""
INSERT INTO detect_run_events (job_id, job_item_id, node_code, event_type, level, message, payload_json)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(job_id, job_item_id, node_code, event_type, level, message, Json(payload or {})),
)
def complete_detect_job_item(self, job_item_id, claim_token, final_status='completed'):
conn = None
cur = None
try:
conn, cur = self.connect()
if not conn or not cur:
return False
cur.execute(
"""
UPDATE detect_job_items
SET status = %s,
finished_at = CURRENT_TIMESTAMP,
updated_at = CURRENT_TIMESTAMP,
lease_expires_at = NULL
WHERE id = %s AND claim_token = %s
RETURNING job_id
""",
(final_status, job_item_id, claim_token),
)
row = cur.fetchone()
if row:
self._refresh_detect_job_status_with_cursor(cur, row[0])
conn.commit()
return bool(row)
except Exception as e:
try:
if conn:
conn.rollback()
except Exception:
pass
logger.error(f"完成检测任务项失败: {e}")
return False
finally:
self.close(conn, cur)
def fail_detect_job_item(self, job_item_id, claim_token, reason=''):
conn = None
cur = None
try:
conn, cur = self.connect()
if not conn or not cur:
return False
cur.execute(
"""
UPDATE detect_job_items
SET status = 'failed',
last_error = %s,
finished_at = CURRENT_TIMESTAMP,
updated_at = CURRENT_TIMESTAMP,
lease_expires_at = NULL
WHERE id = %s AND claim_token = %s
RETURNING job_id
""",
(str(reason or '')[:1000], job_item_id, claim_token),
)
row = cur.fetchone()
if row:
self._refresh_detect_job_status_with_cursor(cur, row[0])
conn.commit()
return bool(row)
except Exception as e:
try:
if conn:
conn.rollback()
except Exception:
pass
logger.error(f"标记检测任务项失败: {e}")
return False
finally:
self.close(conn, cur)
def _refresh_detect_job_status_with_cursor(self, cur, job_id):
cur.execute(
"""
SELECT
count(*) FILTER (WHERE status IN ('pending', 'claimed', 'running')) AS active_count,
count(*) FILTER (WHERE status = 'failed') AS failed_count,
count(*) FILTER (WHERE status IN ('completed', 'blacklisted')) AS done_count
FROM detect_job_items
WHERE job_id = %s
""",
(job_id,),
)
row = cur.fetchone()
active_count = int(row[0] or 0)
failed_count = int(row[1] or 0)
done_count = int(row[2] or 0)
if active_count > 0:
cur.execute(
"UPDATE detect_jobs SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP) WHERE id = %s",
(job_id,),
)
return
final_status = 'completed'
if failed_count > 0 and done_count > 0:
final_status = 'partial_failed'
elif failed_count > 0:
final_status = 'failed'
cur.execute(
"""
UPDATE detect_jobs
SET status = %s,
finished_at = CURRENT_TIMESTAMP,
started_at = COALESCE(started_at, CURRENT_TIMESTAMP)
WHERE id = %s
""",
(final_status, job_id),
)
def fetch_one(self, sql, params=None):
"""
@@ -812,6 +1271,35 @@ class Database:
sql = "UPDATE domains SET detect_status = %s WHERE id = %s"
return self.execute(sql, (status, domain_id))
def recycle_running_domains(self, target_status):
"""
回收异常中断后遗留的“检测中”状态。
:param target_status: 目标状态，通常使用 DETECT_STATUS_FAILED
:return: int - 影响行数
"""
conn, cur = self.connect()
if not conn or not cur:
logger.error("回收检测中状态失败: 无法获取数据库连接")
return 0
try:
cur.execute(
"UPDATE domains SET detect_status = %s WHERE detect_status = %s",
(target_status, DETECT_STATUS_RUNNING)
)
affected = cur.rowcount or 0
conn.commit()
return affected
except Exception as e:
try:
conn.rollback()
except Exception:
pass
logger.error(f"回收检测中状态失败: {e}")
return 0
finally:
self.close(conn, cur)
def update_domain_third_party_status(self, domain_id, field_name, status):
"""
更新第三方平台检测状态。

2
domainCheck/detect/geetest2.py
View File

@@ -3,7 +3,6 @@ import requests, re, json, base64
import io, os, random
import time, cv2, json
from PIL import Image
from functools import partial
from io import BytesIO
@@ -13,7 +12,6 @@ from detect.module.use_ua import randomUA
from detect.module.gap import quekou
import subprocess
subprocess.Popen = partial(subprocess.Popen, encoding='utf-8')
import execjs

31
domainCheck/detect/jucha.py
View File

@@ -15,9 +15,9 @@ import json # JSON数据处理
import os # 操作系统接口
import pickle # 序列化反序列化
import random # 随机数生成
import shutil
import subprocess # 子进程管理
import time # 时间处理
from functools import partial
# 导入第三方库
import requests # HTTP请求库
@@ -33,11 +33,17 @@ from requests.cookies import RequestsCookieJar # Cookie管理
# super().__init__(*args, **kwargs) # 调用父类初始化方法
#
#
# subprocess.Popen = MySubprocessPopen # 替换subprocess.Popen为自定义类
os.environ["EXECJS_RUNTIME"] = "Node" # 设置JavaScript运行时环境为Node.js
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
subprocess.Popen = partial(subprocess.Popen, encoding='utf-8', startupinfo=startupinfo)
_ORIGINAL_POPEN = subprocess.Popen
def create_subprocess(*args, **kwargs):
kwargs.setdefault("encoding", "utf-8")
if hasattr(subprocess, "STARTUPINFO") and hasattr(subprocess, "STARTF_USESHOWWINDOW"):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
kwargs.setdefault("startupinfo", startupinfo)
return _ORIGINAL_POPEN(*args, **kwargs)
def resolve_node_executable() -> str:
@@ -46,6 +52,19 @@ def resolve_node_executable() -> str:
bundled_node = os.path.join(project_dir, "tools", "node-v20.19.4-win-x64", "node.exe")
if os.path.exists(bundled_node):
return bundled_node
detected = shutil.which("node")
if detected:
return detected
fallback_candidates = [
"/usr/bin/node",
"/usr/local/bin/node",
os.path.expanduser("~/.nvm/versions/node/current/bin/node"),
os.path.expanduser("~/.nvm/versions/node/v20.20.2/bin/node"),
"/root/.nvm/versions/node/v20.20.2/bin/node",
]
for candidate in fallback_candidates:
if os.path.exists(candidate):
return candidate
return "node"
def calculate_seed(t: int) -> str: # 计算验证码种子值
@@ -195,7 +214,7 @@ class JC(object): # 聚查网API封装类
return False, f"缺少签名脚本: {jsFile_path}" # 返回失败
try: # 尝试执行Node.js脚本
with subprocess.Popen( # 创建子进程
with create_subprocess( # 创建子进程
[resolve_node_executable(), jsFile_path], # 执行node命令
stdin=subprocess.PIPE, # 标准输入管道
stdout=subprocess.PIPE, # 标准输出管道

35
domainCheck/detect/juming.py
View File

@@ -14,9 +14,9 @@ import json # JSON处理模块
import os # 操作系统接口模块
import pickle # 序列化模块
import random # 随机数生成模块
import shutil
import subprocess # 子进程管理模块
import time # 时间处理模块
from functools import partial
from typing import Optional, Tuple, Dict, List, Any # 类型提示
import requests # HTTP请求库
@@ -34,17 +34,25 @@ SLIDE_OFFSET = 290 # 滑块偏移量
SLIDE_DURATION = 611 # 滑动持续时间
class MySubprocessPopen(subprocess.Popen): # 自定义子进程类
_ORIGINAL_POPEN = subprocess.Popen
class MySubprocessPopen(_ORIGINAL_POPEN): # 自定义子进程类
def __init__(self, *args, **kwargs): # 初始化方法
kwargs['encoding'] = "UTF-8" # 设置编码为UTF-8
super().__init__(*args, **kwargs) # 调用父类初始化
subprocess.Popen = MySubprocessPopen # 替换默认的Popen类
os.environ["EXECJS_RUNTIME"] = "Node" # 设置JavaScript运行时为Node.js
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
subprocess.Popen = partial(subprocess.Popen, encoding='utf-8', startupinfo=startupinfo)
def create_subprocess(*args, **kwargs):
kwargs.setdefault("encoding", "utf-8")
if hasattr(subprocess, "STARTUPINFO") and hasattr(subprocess, "STARTF_USESHOWWINDOW"):
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
kwargs.setdefault("startupinfo", startupinfo)
return MySubprocessPopen(*args, **kwargs)
def resolve_node_executable() -> str:
@@ -53,6 +61,19 @@ def resolve_node_executable() -> str:
bundled_node = os.path.join(project_dir, "tools", "node-v20.19.4-win-x64", "node.exe")
if os.path.exists(bundled_node):
return bundled_node
detected = shutil.which("node")
if detected:
return detected
fallback_candidates = [
"/usr/bin/node",
"/usr/local/bin/node",
os.path.expanduser("~/.nvm/versions/node/current/bin/node"),
os.path.expanduser("~/.nvm/versions/node/v20.20.2/bin/node"),
"/root/.nvm/versions/node/v20.20.2/bin/node",
]
for candidate in fallback_candidates:
if os.path.exists(candidate):
return candidate
return "node"
def calculate_seed(t: int) -> str: # 计算验证码种子值
@@ -230,7 +251,7 @@ class JM(object): # 聚名网API封装类
return False, f"缺少签名脚本: {jsFile_path}" # 返回失败
try: # 异常处理
with subprocess.Popen( # 执行Node.js脚本
with create_subprocess( # 执行Node.js脚本
[resolve_node_executable(), jsFile_path], # 命令和参数
stdin=subprocess.PIPE, # 标准输入
stdout=subprocess.PIPE, # 标准输出

12
domainCheck/detect_options.json
View File

@@ -1,8 +1,4 @@
{
"detect_juziseo": false,
"detect_360_site": true,
"detect_aizhan": true,
"detect_baidu_site": true,
"detect_order": [
"detect_register",
"detect_baidu_site",
@@ -14,7 +10,11 @@
"detect_juziseo"
],
"detect_register": true,
"detect_wayback": true,
"detect_baidu_site": true,
"detect_360_site": true,
"detect_chinaz": true,
"detect_jucha": false
"detect_aizhan": true,
"detect_wayback": true,
"detect_jucha": false,
"detect_juziseo": false
}

1142
domainCheck/detect_worker.py
View File

File diff suppressed because it is too large Load Diff

4
domainCheck/init_database.py
View File

@@ -204,6 +204,7 @@ def init_database():
baidu_site JSONB,
qihu360_site JSONB,
google_site JSONB,
wayback_info JSONB,
chinaz_info JSONB,
aizhan_info JSONB,
juziseo_info JSONB,
@@ -227,6 +228,8 @@ def init_database():
cur.execute("ALTER TABLE domain_detections ADD COLUMN IF NOT EXISTS is_chinese_title BOOLEAN")
# 添加backlink_count_gt_10字段
cur.execute("ALTER TABLE domain_detections ADD COLUMN IF NOT EXISTS backlink_count_gt_10 BOOLEAN")
# 添加wayback_info字段
cur.execute("ALTER TABLE domain_detections ADD COLUMN IF NOT EXISTS wayback_info JSONB")
logger.info("为domain_detections表添加新字段成功")
except Exception as e:
logger.warning(f"为domain_detections表添加新字段失败: {e}")
@@ -240,6 +243,7 @@ def init_database():
cur.execute("COMMENT ON COLUMN domain_detections.baidu_site IS '百度site'")
cur.execute("COMMENT ON COLUMN domain_detections.qihu360_site IS '360 site'")
cur.execute("COMMENT ON COLUMN domain_detections.google_site IS 'Google site'")
cur.execute("COMMENT ON COLUMN domain_detections.wayback_info IS '时光机检测信息'")
cur.execute("COMMENT ON COLUMN domain_detections.chinaz_info IS '站长之家信息'")
cur.execute("COMMENT ON COLUMN domain_detections.aizhan_info IS '爱站网信息'")
cur.execute("COMMENT ON COLUMN domain_detections.juziseo_info IS '桔子SEO信息'")

2
domainCheck/proxy_config.json
View File

@@ -1,7 +1,7 @@
{
"proxy_url": "http://211.101.244.154:18008/getProxy_batch.php?group=B&count=50",
"proxy_enable": true,
"allow_direct": true,
"proxy_url": "http://211.101.244.154:18008/getProxy_batch.php?group=B&count=50",
"proxy_urls": [
"http://211.101.244.154:18008/getProxy_batch.php?group=B&count=50",
"http://211.101.244.154:18008/getProxy_batch.php?group=C&count=50",

2
domainCheck/thread_count.json
View File

@@ -1,3 +1,3 @@
{
"thread_count": "2"
"thread_count": "5"
}