admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first

Browse Source
This commit is contained in:
Your Name
2026-04-16 21:35:47 +08:00
parent ff32aa50bf
commit ebf632e651
86 changed files with 14097 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

272
domain-api/app/services/cluster_runtime_service.py Normal file
View File

@@ -0,0 +1,272 @@
from __future__ import annotations
import json
import socket
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
_RUNTIME_SCHEMA_SQL = """
CREATE TABLE IF NOT EXISTS detect_worker_nodes (
node_code VARCHAR(64) PRIMARY KEY,
region VARCHAR(32) NOT NULL DEFAULT 'unknown',
role VARCHAR(32) NOT NULL DEFAULT 'worker',
hostname VARCHAR(255) NOT NULL DEFAULT '',
ip VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'unknown',
worker_version VARCHAR(32) NOT NULL DEFAULT '',
current_load INTEGER NOT NULL DEFAULT 0,
metadata_json JSONB,
last_heartbeat_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS detect_jobs (
id BIGSERIAL PRIMARY KEY,
job_code VARCHAR(64) NOT NULL UNIQUE,
source VARCHAR(64) NOT NULL DEFAULT 'manual',
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
remark TEXT NOT NULL DEFAULT '',
created_by VARCHAR(64) NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
started_at TIMESTAMP,
finished_at TIMESTAMP
);
CREATE TABLE IF NOT EXISTS detect_job_items (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
domain_id BIGINT NOT NULL,
status VARCHAR(32) NOT NULL DEFAULT 'pending',
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
claim_token VARCHAR(64) NOT NULL DEFAULT '',
lease_expires_at TIMESTAMP,
attempt_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT NOT NULL DEFAULT '',
result_version VARCHAR(64) NOT NULL DEFAULT '',
started_at TIMESTAMP,
finished_at TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
);
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
ON detect_job_items(status, lease_expires_at);
CREATE TABLE IF NOT EXISTS detect_run_events (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
job_item_id BIGINT REFERENCES detect_job_items(id) ON DELETE SET NULL,
node_code VARCHAR(64) NOT NULL DEFAULT '',
event_type VARCHAR(64) NOT NULL DEFAULT '',
level VARCHAR(16) NOT NULL DEFAULT 'info',
message TEXT NOT NULL DEFAULT '',
payload_json JSONB,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_detect_run_events_job_created
ON detect_run_events(job_id, created_at DESC);
CREATE TABLE IF NOT EXISTS detect_sync_records (
id BIGSERIAL PRIMARY KEY,
sync_type VARCHAR(32) NOT NULL DEFAULT '',
source_region VARCHAR(32) NOT NULL DEFAULT '',
target_region VARCHAR(32) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
payload_json JSONB,
error_message TEXT NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
"""
def _resolve_local_ip() -> str:
try:
return socket.gethostbyname(socket.gethostname())
except Exception:
return ""
def _decode_json(value: object) -> dict:
if isinstance(value, dict):
return value
if value in (None, ""):
return {}
try:
return json.loads(value)
except Exception:
return {}
def ensure_runtime_schema() -> None:
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_RUNTIME_SCHEMA_SQL)
conn.commit()
def register_node_heartbeat(
*,
node_code: str,
region: str,
role: str,
status: str,
current_load: int = 0,
worker_version: str = "0.1.0",
metadata: dict | None = None,
) -> None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO detect_worker_nodes (
node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
ON CONFLICT (node_code) DO UPDATE SET
region = EXCLUDED.region,
role = EXCLUDED.role,
hostname = EXCLUDED.hostname,
ip = EXCLUDED.ip,
status = EXCLUDED.status,
worker_version = EXCLUDED.worker_version,
current_load = EXCLUDED.current_load,
metadata_json = EXCLUDED.metadata_json,
last_heartbeat_at = CURRENT_TIMESTAMP,
update_time = CURRENT_TIMESTAMP
""",
(
node_code,
region,
role,
socket.gethostname(),
_resolve_local_ip(),
status,
worker_version,
max(0, int(current_load or 0)),
json.dumps(metadata or {}, ensure_ascii=False),
),
)
conn.commit()
def register_local_control_heartbeat() -> None:
register_node_heartbeat(
node_code=settings.node_code,
region=settings.node_region,
role=settings.node_role,
status="online",
current_load=0,
metadata={
"service": "domain-api",
"api_host": settings.api_host,
"api_port": settings.api_port,
"worker_mode": settings.worker_mode,
"updated_at": datetime.now().isoformat(timespec="seconds"),
},
)
def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None) -> str:
status = str(raw_status or "").strip() or "unknown"
if not last_heartbeat_at:
return status
now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now()
age = now - last_heartbeat_at
if age > timedelta(minutes=5):
return "offline"
if age > timedelta(seconds=90):
return "stale"
return status
def get_cluster_snapshot() -> dict:
register_local_control_heartbeat()
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at
FROM detect_worker_nodes
ORDER BY
CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END,
region ASC,
role ASC,
node_code ASC
LIMIT 100
"""
)
rows = cur.fetchall()
cur.execute("SELECT count(*) FROM detect_jobs")
jobs_total = cur.fetchone()[0]
cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')")
active_items = cur.fetchone()[0]
nodes = [
{
"node_code": row[0],
"region": row[1],
"role": row[2],
"hostname": row[3],
"ip": row[4],
"status": _normalize_node_status(row[5], row[9]),
"worker_version": row[6],
"current_load": row[7],
"metadata": _decode_json(row[8]),
"last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "",
}
for row in rows
]
status_counts: dict[str, int] = {}
role_counts: dict[str, int] = {}
region_counts: dict[str, int] = {}
busy_nodes: list[str] = []
stale_nodes: list[str] = []
offline_nodes: list[str] = []
online_worker_nodes = 0
online_control_nodes = 0
for node in nodes:
node_status = str(node.get("status") or "unknown")
node_role = str(node.get("role") or "unknown")
node_region = str(node.get("region") or "unknown")
node_code = str(node.get("node_code") or "")
status_counts[node_status] = status_counts.get(node_status, 0) + 1
role_counts[node_role] = role_counts.get(node_role, 0) + 1
region_counts[node_region] = region_counts.get(node_region, 0) + 1
if node_status == "busy":
busy_nodes.append(node_code)
if node_status == "stale":
stale_nodes.append(node_code)
if node_status == "offline":
offline_nodes.append(node_code)
if node_role == "worker" and node_status in {"online", "busy"}:
online_worker_nodes += 1
if node_role == "control" and node_status in {"online", "busy"}:
online_control_nodes += 1
return {
"nodes": nodes,
"nodes_total": len(nodes),
"jobs_total": jobs_total,
"active_job_items": active_items,
"summary": {
"status_counts": status_counts,
"role_counts": role_counts,
"region_counts": region_counts,
"busy_nodes": busy_nodes,
"stale_nodes": stale_nodes,
"offline_nodes": offline_nodes,
"online_worker_nodes": online_worker_nodes,
"online_control_nodes": online_control_nodes,
},
}

558
domain-api/app/services/detect_job_service.py Normal file
View File

@@ -0,0 +1,558 @@
from __future__ import annotations
import json
from datetime import datetime
import math
from uuid import uuid4
from app.core.config import settings
from app.core.db import get_db
ACTIVE_JOB_STATUSES = ("pending", "running")
def _selection_sql() -> str:
return """
SELECT id
FROM domains
WHERE
detect_status IN (0, 4)
OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
ORDER BY id ASC
LIMIT %s
"""
def _format_time(value: datetime | None) -> str:
return value.isoformat(sep=" ", timespec="seconds") if value else ""
def _decode_payload(value: object) -> dict:
if isinstance(value, dict):
return value
if value in (None, ""):
return {}
try:
return json.loads(value)
except Exception:
return {}
def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]:
if not events:
return "", []
latest_payload = events[0].get("payload") or {}
latest_cycle_token = str(latest_payload.get("cycle_token") or "").strip()
if latest_cycle_token:
current_cycle = []
for event in events:
payload = event.get("payload") or {}
event_cycle = str(payload.get("cycle_token") or "").strip()
if event_cycle == latest_cycle_token:
current_cycle.append(event)
return latest_cycle_token, current_cycle
cycle_token = ""
anchor_index = -1
for index, event in enumerate(events):
payload = event.get("payload") or {}
event_type = str(event.get("event_type") or "")
candidate = str(payload.get("cycle_token") or "").strip()
if candidate and event_type in {"job_dispatch_sent", "job_dispatch_requested", "job_dispatch_failed", "job_dispatch_rejected"}:
cycle_token = candidate
anchor_index = index
break
if not cycle_token:
return "", events
current_cycle = []
for index, event in enumerate(events[: anchor_index + 1]):
payload = event.get("payload") or {}
event_cycle = str(payload.get("cycle_token") or "").strip()
if index == anchor_index or event_cycle == cycle_token:
current_cycle.append(event)
return cycle_token, current_cycle
def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
job_id = job_row[0]
cur.execute(
"""
SELECT status, count(*)
FROM detect_job_items
WHERE job_id = %s
GROUP BY status
""",
(job_id,),
)
counts = {status: int(count) for status, count in cur.fetchall()}
cur.execute(
"""
SELECT COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code,
status,
count(*)
FROM detect_job_items
WHERE job_id = %s
GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned'), status
ORDER BY node_code ASC, status ASC
""",
(job_id,),
)
node_buckets: dict[str, dict] = {}
for node_code, status, count in cur.fetchall():
bucket = node_buckets.setdefault(
node_code,
{
"node_code": node_code,
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
},
)
field_name = f"items_{status}"
if field_name in bucket:
bucket[field_name] += int(count)
bucket["items_total"] += int(count)
cur.execute(
"""
SELECT node_code, event_type, level, message, payload_json, created_at
FROM detect_run_events
WHERE job_id = %s
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(job_id, max(1, int(event_limit or 20))),
)
events = [
{
"node_code": item[0],
"event_type": item[1],
"level": item[2],
"message": item[3],
"payload": _decode_payload(item[4]),
"created_at": _format_time(item[5]),
}
for item in cur.fetchall()
]
cycle_token, current_cycle_events = _extract_current_cycle_events(events)
total = sum(counts.values())
terminal = int(counts.get("completed", 0)) + int(counts.get("blacklisted", 0)) + int(counts.get("failed", 0))
return {
"job_id": job_id,
"job_code": job_row[1],
"source": job_row[2],
"status": job_row[3],
"created_by": job_row[4],
"created_at": _format_time(job_row[5]),
"started_at": _format_time(job_row[6]),
"finished_at": _format_time(job_row[7]),
"items_total": total,
"items_pending": int(counts.get("pending", 0)),
"items_claimed": int(counts.get("claimed", 0)),
"items_running": int(counts.get("running", 0)),
"items_completed": int(counts.get("completed", 0)),
"items_blacklisted": int(counts.get("blacklisted", 0)),
"items_failed": int(counts.get("failed", 0)),
"items_terminal": terminal,
"progress_percent": round((terminal / total) * 100, 2) if total else 0,
"node_stats": list(node_buckets.values()),
"recent_events": events,
"latest_event": events[0] if events else None,
"current_cycle_token": cycle_token,
"current_cycle_events": current_cycle_events,
"latest_cycle_event": current_cycle_events[0] if current_cycle_events else None,
}
def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
FROM detect_jobs
WHERE id = %s
LIMIT 1
""",
(int(job_id),),
)
row = cur.fetchone()
if not row:
return None
return _fetch_job_summary(cur, row, event_limit=event_limit)
def get_active_detect_job_summary(event_limit: int = 20) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
FROM detect_jobs
WHERE status IN ('pending', 'running')
ORDER BY created_at DESC, id DESC
LIMIT 1
"""
)
row = cur.fetchone()
if not row:
return None
return _fetch_job_summary(cur, row, event_limit=event_limit)
def list_detect_jobs(limit: int = 20) -> list[dict]:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
FROM detect_jobs
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(max(1, min(int(limit or 20), 100)),),
)
rows = cur.fetchall()
return [_fetch_job_summary(cur, row, event_limit=10) for row in rows]
def get_detect_queue_health(window_minutes: int = 15) -> dict:
window_minutes = max(5, min(int(window_minutes or 15), 120))
active_job = get_active_detect_job_summary(event_limit=10)
if not active_job:
return {
"window_minutes": window_minutes,
"has_active_job": False,
"job": None,
"queue": {
"items_total": 0,
"pending": 0,
"claimed": 0,
"running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
"terminal": 0,
"terminal_percent": 0,
"oldest_pending_at": "",
"oldest_pending_age_minutes": 0,
"nearest_lease_expiry_at": "",
"overdue_leases": 0,
"expiring_soon_leases": 0,
},
"throughput": {
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
},
"nodes": [],
}
job_id = int(active_job["job_id"])
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
MIN(create_time) FILTER (WHERE status = 'pending') AS oldest_pending_at,
MIN(lease_expires_at) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL) AS nearest_lease_expiry_at,
COUNT(*) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at < CURRENT_TIMESTAMP) AS overdue_leases,
COUNT(*) FILTER (
WHERE status IN ('claimed', 'running')
AND lease_expires_at IS NOT NULL
AND lease_expires_at >= CURRENT_TIMESTAMP
AND lease_expires_at < CURRENT_TIMESTAMP + interval '5 minutes'
) AS expiring_soon_leases
FROM detect_job_items
WHERE job_id = %s
""",
(job_id,),
)
lease_row = cur.fetchone()
cur.execute(
"""
SELECT
COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code,
COUNT(*) AS processed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
FROM detect_run_events
WHERE job_id = %s
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
GROUP BY COALESCE(NULLIF(node_code, ''), 'unknown')
ORDER BY processed_recent DESC, node_code ASC
""",
(job_id, window_minutes),
)
throughput_rows = cur.fetchall()
oldest_pending_at = _format_time(lease_row[0]) if lease_row and lease_row[0] else ""
nearest_lease_expiry_at = _format_time(lease_row[1]) if lease_row and lease_row[1] else ""
oldest_pending_age_minutes = 0
if lease_row and lease_row[0]:
oldest_pending_age_minutes = max(0, int((datetime.now() - lease_row[0]).total_seconds() // 60))
node_map = {
str(item.get("node_code") or "unknown"): {
"node_code": str(item.get("node_code") or "unknown"),
"items_total": int(item.get("items_total", 0) or 0),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
}
for item in active_job.get("node_stats") or []
}
total_processed_recent = 0
total_completed_recent = 0
total_blacklisted_recent = 0
total_failed_recent = 0
for row in throughput_rows:
node_code = str(row[0] or "unknown")
bucket = node_map.setdefault(
node_code,
{
"node_code": node_code,
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
},
)
processed_recent = int(row[1] or 0)
completed_recent = int(row[2] or 0)
blacklisted_recent = int(row[3] or 0)
failed_recent = int(row[4] or 0)
bucket["processed_recent"] = processed_recent
bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2)
bucket["completed_recent"] = completed_recent
bucket["blacklisted_recent"] = blacklisted_recent
bucket["failed_recent"] = failed_recent
total_processed_recent += processed_recent
total_completed_recent += completed_recent
total_blacklisted_recent += blacklisted_recent
total_failed_recent += failed_recent
nodes = sorted(
node_map.values(),
key=lambda item: (
-int(item.get("processed_recent", 0) or 0),
-int(item.get("items_running", 0) or 0),
str(item.get("node_code") or ""),
),
)
items_total = int(active_job.get("items_total", 0) or 0)
terminal = int(active_job.get("items_terminal", 0) or 0)
return {
"window_minutes": window_minutes,
"has_active_job": True,
"job": {
"job_id": job_id,
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
"progress_percent": active_job.get("progress_percent", 0),
},
"queue": {
"items_total": items_total,
"pending": int(active_job.get("items_pending", 0) or 0),
"claimed": int(active_job.get("items_claimed", 0) or 0),
"running": int(active_job.get("items_running", 0) or 0),
"completed": int(active_job.get("items_completed", 0) or 0),
"blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
"failed": int(active_job.get("items_failed", 0) or 0),
"terminal": terminal,
"terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0,
"oldest_pending_at": oldest_pending_at,
"oldest_pending_age_minutes": oldest_pending_age_minutes,
"nearest_lease_expiry_at": nearest_lease_expiry_at,
"overdue_leases": int(lease_row[2] or 0) if lease_row else 0,
"expiring_soon_leases": int(lease_row[3] or 0) if lease_row else 0,
},
"throughput": {
"processed_recent": total_processed_recent,
"processed_per_minute": round(total_processed_recent / window_minutes, 2),
"completed_recent": total_completed_recent,
"blacklisted_recent": total_blacklisted_recent,
"failed_recent": total_failed_recent,
},
"nodes": nodes,
}
def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict:
queue_health = queue_health or get_detect_queue_health(window_minutes=15)
target_finish_hours = max(1, min(int(target_finish_hours or 6), 72))
online_worker_nodes = max(0, int(online_worker_nodes or 0))
if not queue_health.get("has_active_job"):
return {
"has_active_job": False,
"online_worker_nodes": online_worker_nodes,
"target_finish_hours": target_finish_hours,
"estimated_hours_remaining": 0,
"recommended_total_workers": max(1, online_worker_nodes),
"recommended_additional_workers": 0,
"current_processed_per_hour": 0,
"pending_items": 0,
"terminal_items": 0,
"summary": "当前没有活跃任务，无需扩容建议。",
}
queue = queue_health.get("queue") or {}
throughput = queue_health.get("throughput") or {}
pending_items = int(queue.get("pending", 0) or 0)
claimed_items = int(queue.get("claimed", 0) or 0)
running_items = int(queue.get("running", 0) or 0)
remaining_items = pending_items + claimed_items + running_items
current_processed_per_hour = round(float(throughput.get("processed_per_minute", 0) or 0) * 60, 2)
estimated_hours_remaining = round((remaining_items / current_processed_per_hour), 2) if current_processed_per_hour > 0 else 0
recommended_total_workers = max(1, online_worker_nodes or 1)
recommended_additional_workers = 0
if remaining_items > 0 and target_finish_hours > 0:
required_per_hour = remaining_items / target_finish_hours
if current_processed_per_hour > 0 and max(1, online_worker_nodes) > 0:
per_worker_per_hour = current_processed_per_hour / max(1, online_worker_nodes)
recommended_total_workers = max(1, int(math.ceil(required_per_hour / per_worker_per_hour)))
recommended_additional_workers = max(0, recommended_total_workers - online_worker_nodes)
elif remaining_items > 0:
recommended_total_workers = max(1, online_worker_nodes or 1)
recommended_additional_workers = 0
summary = (
f"当前在线 Worker {online_worker_nodes} 台，近窗吞吐约 {current_processed_per_hour} 项/小时，"
f"剩余待处理约 {remaining_items} 项，预计还需 {estimated_hours_remaining} 小时。"
)
if recommended_additional_workers > 0:
summary = (
f"{summary} 若希望在 {target_finish_hours} 小时内收敛，建议总 Worker 数达到 "
f"{recommended_total_workers} 台，至少再加 {recommended_additional_workers} 台。"
)
else:
summary = f"{summary} 按当前目标 {target_finish_hours} 小时看，现有 Worker 数量暂时够用。"
return {
"has_active_job": True,
"online_worker_nodes": online_worker_nodes,
"target_finish_hours": target_finish_hours,
"estimated_hours_remaining": estimated_hours_remaining,
"recommended_total_workers": recommended_total_workers,
"recommended_additional_workers": recommended_additional_workers,
"current_processed_per_hour": current_processed_per_hour,
"pending_items": pending_items,
"remaining_items": remaining_items,
"terminal_items": int(queue.get("terminal", 0) or 0),
"summary": summary,
}
def append_detect_job_event(
job_id: int,
*,
event_type: str,
message: str,
level: str = "info",
payload: dict | None = None,
node_code: str | None = None,
) -> None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
""",
(
int(job_id),
node_code or settings.node_code,
str(event_type or "").strip() or "info",
str(level or "info").strip() or "info",
str(message or "").strip(),
json.dumps(payload or {}, ensure_ascii=False),
),
)
conn.commit()
def create_detect_job_if_needed(limit: int = 1000, created_by: str = "system") -> dict | None:
existing = get_active_detect_job_summary()
if existing:
return existing
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_selection_sql(), (max(1, int(limit or 1000)),))
domain_ids = [row[0] for row in cur.fetchall()]
if not domain_ids:
conn.rollback()
return None
job_code = f"detect-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}"
plan_hash = uuid4().hex
cur.execute(
"""
INSERT INTO detect_jobs (job_code, source, plan_hash, status, remark, created_by)
VALUES (%s, %s, %s, 'pending', %s, %s)
RETURNING id
""",
(
job_code,
"api-start",
plan_hash,
f"API 创建检测任务，待检测域名 {len(domain_ids)} 个",
created_by,
),
)
job_id = cur.fetchone()[0]
item_rows = [(job_id, domain_id) for domain_id in domain_ids]
cur.executemany(
"""
INSERT INTO detect_job_items (job_id, domain_id, status)
VALUES (%s, %s, 'pending')
ON CONFLICT (job_id, domain_id) DO NOTHING
""",
item_rows,
)
cur.execute(
"""
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
""",
(
job_id,
settings.node_code,
"job_created",
"info",
f"创建检测任务 {job_code}，共 {len(domain_ids)} 个域名",
'{"count": %s}' % len(domain_ids),
),
)
conn.commit()
return get_active_detect_job_summary()

425
domain-api/app/services/detect_run_service.py Normal file
View File

@@ -0,0 +1,425 @@
from __future__ import annotations
from datetime import datetime
from uuid import uuid4
from app.core.files import load_detect_records, save_detect_records, tail_lines
_MAX_LOG_LINES = 240
_LOG_TAIL_LINES = 1200
_ACTIVE_STATUSES = {"starting", "running", "stopping"}
_TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S")
def _now() -> str:
return datetime.now().isoformat(sep=" ", timespec="seconds")
def _save(records: list[dict]) -> None:
save_detect_records(records)
def _load() -> list[dict]:
return load_detect_records()
def _capture_worker_logs(max_lines: int = _LOG_TAIL_LINES) -> list[str]:
lines = tail_lines("detect_worker.log", max_lines=max_lines)
return lines[-max_lines:]
def _parse_time(raw: str | None) -> datetime | None:
if not raw:
return None
text = str(raw).strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
pass
for fmt in _TIMESTAMP_FORMATS:
try:
return datetime.strptime(text, fmt)
except ValueError:
continue
return None
def _extract_log_time(line: str) -> datetime | None:
if len(line) < 19:
return None
candidates = [line[:26], line[:19]]
for candidate in candidates:
for fmt in _TIMESTAMP_FORMATS:
if len(candidate) != len(datetime.now().strftime(fmt)):
continue
try:
return datetime.strptime(candidate, fmt)
except ValueError:
continue
return None
def _filter_logs_since(lines: list[str], started_at: str | None) -> list[str]:
if not started_at:
return lines[-_MAX_LOG_LINES:]
started_time = _parse_time(started_at)
if not started_time:
return lines[-_MAX_LOG_LINES:]
filtered = [line for line in lines if (_extract_log_time(line) or started_time) >= started_time]
if filtered:
return filtered[-_MAX_LOG_LINES:]
return lines[-_MAX_LOG_LINES:]
def _merge_logs(existing: list[str] | None, current: list[str]) -> list[str]:
merged = list(existing or [])
for line in current:
if line not in merged[-40:]:
merged.append(line)
continue
if not merged or merged[-1] != line:
merged.append(line)
return merged[-_MAX_LOG_LINES:]
def _find_active(records: list[dict]) -> dict | None:
return next((item for item in records if item.get("status") in _ACTIVE_STATUSES), None)
def _same_session(active: dict | None, runtime: dict) -> bool:
if not active:
return False
runtime_started_at = str(runtime.get("latest_start_time", "") or "").strip()
active_started_at = str(active.get("started_at", "") or "").strip()
if not runtime_started_at or not active_started_at:
return True
runtime_started = _parse_time(runtime_started_at)
active_started = _parse_time(active_started_at)
if not runtime_started or not active_started:
return True
return abs((runtime_started - active_started).total_seconds()) < 3
def _latest_matching_log(log_lines: list[str], keywords: tuple[str, ...]) -> str:
for line in reversed(log_lines or []):
if any(keyword in line for keyword in keywords):
return line
return ""
def _sync_phase_history(record: dict, phase_label: str, phase_detail: str) -> None:
history = list(record.get("phase_history") or [])
current = {
"at": _now(),
"label": phase_label or "-",
"detail": phase_detail or "",
}
if history:
latest = history[-1]
if latest.get("label") == current["label"] and latest.get("detail") == current["detail"]:
return
history.append(current)
record["phase_history"] = history[-20:]
def _phase_from_runtime(status: str, runtime: dict, progress: dict, log_lines: list[str], active_job: dict | None = None) -> tuple[str, str]:
active_job = active_job or {}
if status == "starting":
return "启动中", "正在拉起检测服务并等待 Worker 就绪"
if status == "stopping":
return "停止中", "已发送停止请求，等待 Worker 退出并归档日志"
if status == "completed":
terminal = int(active_job.get("items_terminal", 0) or 0)
return "已完成", f"本轮检测已自然完成，本次累计处理 {terminal} 个任务项"
if status == "partial_failed":
failed = int(active_job.get("items_failed", 0) or 0)
terminal = int(active_job.get("items_terminal", 0) or 0)
return "部分失败", f"本轮检测已结束，其中失败 {failed} 个，累计处理 {terminal} 个任务项"
if status == "failed":
return "失败", runtime.get("message") or "Worker 异常退出，请检查日志"
if status == "stopped":
return "已停止", runtime.get("message") or "检测服务已停止"
if runtime.get("running"):
proxy_runtime_state = str(runtime.get("proxy_runtime_state", "") or "").strip()
proxy_runtime_detail = str(runtime.get("proxy_runtime_detail", "") or "").strip()
running = (progress or {}).get("running", 0)
pending = (progress or {}).get("pending", 0)
phase_log = _latest_matching_log(
log_lines,
(
"Connection refused",
"Read timed out",
"ConnectTimeout",
"HTTPSConnectionPool",
"WaybackDetector",
"域名检测任务完成",
"当前批次检测完成",
"当前实际线程数量",
"开始创建线程",
"获取到",
"开始检测，刷新代理池",
"刷新代理池",
"没有需要检测的域名",
"开始执行域名检测任务",
),
)
if proxy_runtime_state == "blocked_no_proxy":
return "等待代理", proxy_runtime_detail or "代理池当前无可用代理，且未允许直连"
if proxy_runtime_state == "degraded_direct":
return "降级直连", proxy_runtime_detail or "代理池暂无可用代理，当前使用直连继续执行"
if "外部依赖异常，步骤降级继续执行" in phase_log:
return "外部站点异常", "外部依赖当前波动，系统已按降级策略继续执行并保留人工复核"
if "WaybackDetector" in phase_log or "web.archive.org" in phase_log:
return "外部站点异常", "时光机依赖当前访问异常，任务仍在继续，建议关注网络或代理策略"
if any(keyword in phase_log for keyword in ("Connection refused", "Read timed out", "ConnectTimeout", "HTTPSConnectionPool")):
return "网络波动", phase_log
if "域名检测任务完成" in phase_log:
return "完成归档", "本轮检测已处理完成，正在等待下一轮任务或归档最终日志"
if "当前批次检测完成" in phase_log:
return "批次完成", phase_log
if "当前实际线程数量" in phase_log or running > 0:
if running > 0:
return "检测中", f"Worker 正在处理 {running} 个检测任务"
return "检测中", phase_log
if "开始创建线程" in phase_log:
return "建线程中", phase_log
if "获取到" in phase_log:
return "取任务中", phase_log
if "刷新代理池" in phase_log:
return "刷新代理池", phase_log
if "没有需要检测的域名" in phase_log:
return "空闲等待", "Worker 在线，当前没有待检测域名"
if "开始执行域名检测任务" in phase_log:
return "准备检测", phase_log
if running > 0:
return "检测中", f"Worker 正在处理 {running} 个检测任务"
if pending > 0:
return "取任务中", f"Worker 在线，等待或领取待检测域名，当前剩余 {pending} 个"
return "运行中", "Worker 在线，当前没有活跃检测任务"
return "已停止", runtime.get("message") or "检测服务已停止"
def _sync_record(
record: dict,
*,
status: str,
message: str,
runtime: dict,
progress: dict,
settings_summary: dict,
log_lines: list[str],
active_job: dict | None = None,
) -> dict:
record["status"] = status
record["message"] = message
record["runtime"] = runtime
record["progress"] = progress
record["settings_summary"] = settings_summary
record["updated_at"] = _now()
if not record.get("started_at"):
record["started_at"] = runtime.get("latest_start_time") or record["updated_at"]
session_logs = _merge_logs(record.get("logs"), _filter_logs_since(log_lines, record.get("started_at")))
record["logs"] = session_logs
record["active_job"] = active_job or {}
record["phase_label"], record["phase_detail"] = _phase_from_runtime(status, runtime, progress, session_logs, active_job)
_sync_phase_history(record, record.get("phase_label", ""), record.get("phase_detail", ""))
if status in {"stopped", "failed", "completed", "partial_failed"} and not record.get("completed_at"):
record["completed_at"] = _now()
if status in _ACTIVE_STATUSES:
record["completed_at"] = ""
return record
def create_detect_run_snapshot(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict:
records = _load()
active = _find_active(records)
current_logs = _capture_worker_logs()
if active:
if active.get("status") == "stopping" and runtime.get("running"):
active["status"] = "running"
_sync_record(
active,
status=active.get("status", "starting"),
message=message,
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=current_logs,
)
_save(records)
return dict(active)
initial_started_at = runtime.get("latest_start_time") or _now()
record = {
"run_id": uuid4().hex,
"status": "starting",
"message": message,
"created_at": _now(),
"updated_at": _now(),
"started_at": initial_started_at,
"completed_at": "",
"runtime": runtime,
"progress": progress,
"settings_summary": settings_summary,
"phase_label": "",
"phase_detail": "",
"phase_history": [],
"logs": [],
}
_sync_record(
record,
status="starting",
message=message,
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=current_logs,
)
records.insert(0, record)
_save(records)
return dict(record)
def finalize_detect_run(message: str, runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> dict | None:
records = _load()
target = _find_active(records)
if not target:
return None
final_status = "stopped" if target.get("status") == "stopping" else "failed"
_sync_record(
target,
status=final_status,
message=message,
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=_capture_worker_logs(),
active_job=active_job,
)
_save(records)
return dict(target)
def sync_detect_runs(runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> list[dict]:
records = _load()
active = _find_active(records)
current_logs = _capture_worker_logs()
active_job = active_job or {}
runtime_detecting = bool(runtime.get("detecting", False))
active_job_status = str(active_job.get("status", "") or "").strip()
active_job_open = active_job_status in {"pending", "running"}
execution_active = runtime_detecting or active_job_open or int((progress or {}).get("running", 0) or 0) > 0
if runtime.get("running") and execution_active:
if active and not _same_session(active, runtime):
_sync_record(
active,
status="stopped",
message="检测服务已重启，上一轮会话已归档",
runtime=active.get("runtime") or runtime,
progress=active.get("progress") or progress,
settings_summary=active.get("settings_summary") or settings_summary,
log_lines=current_logs,
active_job=active.get("active_job") or active_job,
)
active = None
if active:
next_status = "running" if active.get("status") != "stopping" else "stopping"
_sync_record(
active,
status=next_status,
message=runtime.get("message") or active.get("message") or "检测服务运行中",
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=current_logs,
active_job=active_job,
)
else:
started_at = runtime.get("latest_start_time") or _now()
record = {
"run_id": uuid4().hex,
"status": "running",
"message": runtime.get("message") or "检测服务运行中",
"created_at": _now(),
"updated_at": _now(),
"started_at": started_at,
"completed_at": "",
"runtime": runtime,
"progress": progress,
"settings_summary": settings_summary,
"phase_label": "",
"phase_detail": "",
"phase_history": [],
"logs": [],
"active_job": active_job,
}
_sync_record(
record,
status="running",
message=record["message"],
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=current_logs,
active_job=active_job,
)
records.insert(0, record)
elif active:
if runtime.get("running") and not execution_active:
if active.get("status") == "stopping":
final_status = "stopped"
final_message = runtime.get("message") or "检测任务已停止，Worker 保持待命"
elif active_job_status == "partial_failed":
final_status = "partial_failed"
final_message = "检测任务已结束，存在部分失败项"
elif active_job_status == "failed":
final_status = "failed"
final_message = "检测任务已结束，任务结果为失败"
else:
final_status = "completed"
final_message = "检测任务已自然完成，Worker 保持待命"
else:
final_status = "stopped" if active.get("status") == "stopping" else "failed"
final_message = runtime.get("message") or ("检测服务已停止" if final_status == "stopped" else "检测服务异常退出")
_sync_record(
active,
status=final_status,
message=final_message,
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=current_logs,
active_job=active_job,
)
if records:
records[0]["logs"] = _merge_logs(
records[0].get("logs"),
_filter_logs_since(current_logs, records[0].get("started_at")),
)
_save(records)
return records
def mark_detect_run_stopping(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict | None:
records = _load()
target = _find_active(records)
if not target:
return None
_sync_record(
target,
status="stopping",
message=message,
runtime=runtime,
progress=progress,
settings_summary=settings_summary,
log_lines=_capture_worker_logs(),
active_job=target.get("active_job") or {},
)
_save(records)
return dict(target)

356
domain-api/app/services/detect_service.py
View File

@@ -1,15 +1,275 @@
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from app.core.db import get_db
from app.core.files import tail_lines
from app.core.redis_client import get_redis
from app.services.runtime_settings_service import get_runtime_settings
from app.services.settings_service import get_settings_payload
from app.services.detect_run_service import sync_detect_runs
from app.services.detect_job_service import get_active_detect_job_summary
from app.services.settings_service import get_settings_payload, resolve_thread_count
from app.services.sync_record_service import append_detect_result_projection_if_changed
from app.services.worker_control_service import detect_worker_runtime
_PROXY_COUNT_RE = re.compile(r"当前可用代理数[:：]\s*(\d+)")
_THREAD_COUNT_RE = re.compile(r"当前实际线程数量[:：]\s*(\d+)\s*/\s*(\d+)")
_RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state"
_TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S")
def _extract_dependency_alerts(lines: list[str]) -> list[dict]:
alerts: list[dict] = []
recent_lines = lines[-120:] if lines else []
degraded_line = ""
for line in reversed(recent_lines):
if "外部依赖异常，步骤降级继续执行" in line:
degraded_line = line
alerts.append(
{
"kind": "dependency_degraded",
"level": "warning",
"title": "外部依赖降级继续",
"detail": line,
}
)
break
if degraded_line:
return alerts
for line in reversed(recent_lines):
if "WaybackDetector" in line or "web.archive.org" in line:
alerts.append(
{
"kind": "wayback",
"level": "warning",
"title": "时光机依赖异常",
"detail": line,
}
)
break
for line in reversed(recent_lines):
if any(keyword in line for keyword in ("HTTPSConnectionPool", "Connection refused", "Read timed out", "ConnectTimeout")):
alerts.append(
{
"kind": "network",
"level": "warning",
"title": "外部网络波动",
"detail": line,
}
)
break
return alerts
def _extract_available_proxy_count(lines: list[str]) -> int:
for line in reversed(lines):
match = _PROXY_COUNT_RE.search(line)
if match:
return int(match.group(1))
return 0
def _extract_active_thread_snapshot(lines: list[str]) -> dict:
for line in reversed(lines):
match = _THREAD_COUNT_RE.search(line)
if match:
return {
"active": int(match.group(1)),
"max": int(match.group(2)),
}
return {"active": 0, "max": 0}
def _parse_time(raw: str | None) -> datetime | None:
if not raw:
return None
text = str(raw).strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
pass
for fmt in _TIMESTAMP_FORMATS:
try:
return datetime.strptime(text, fmt)
except ValueError:
continue
return None
def _extract_log_time(line: str) -> datetime | None:
if len(line) < 19:
return None
candidates = [line[:26], line[:19]]
for candidate in candidates:
for fmt in _TIMESTAMP_FORMATS:
if len(candidate) != len(datetime.now().strftime(fmt)):
continue
try:
return datetime.strptime(candidate, fmt)
except ValueError:
continue
return None
def _filter_lines_since(lines: list[str], started_at: str | None) -> list[str]:
started_time = _parse_time(started_at)
if not started_time:
return lines
filtered = [line for line in lines if (_extract_log_time(line) or started_time) >= started_time]
return filtered or lines
def _recent_event(lines: list[str]) -> str:
interesting_keywords = (
"开始检测",
"获取到",
"当前批次检测完成",
"域名检测任务完成",
"代理池刷新完成",
"没有需要检测的域名",
"检测已停止",
)
for line in reversed(lines):
if any(keyword in line for keyword in interesting_keywords):
return line
return ""
def _load_runtime_state() -> dict:
try:
redis_client = get_redis()
raw = redis_client.get(_RUNTIME_STATE_KEY)
if not raw:
return {}
data = json.loads(raw)
return data if isinstance(data, dict) else {}
except Exception:
return {}
def _normalize_recent_warning(runtime_state: dict, recent_lines: list[str], available_proxy_count: int) -> str:
runtime_warning = str(runtime_state.get("recent_warning", "") or "").strip()
if runtime_warning:
if runtime_warning in {"未刷新", "代理未启用", "未配置代理池链接"}:
return ""
if available_proxy_count > 0 and ("无可用代理" in runtime_warning or "未取到可用代理数据" in runtime_warning):
return ""
return runtime_warning
recent_proxy_warning = next(
(line for line in reversed(recent_lines) if "代理池刷新失败" in line or "无可用代理" in line or "Redis订阅失败" in line),
"",
)
if available_proxy_count > 0 and "无可用代理" in recent_proxy_warning:
return ""
return recent_proxy_warning
def _build_proxy_runtime_snapshot(settings_payload: dict, runtime_state: dict, available_proxy_count: int) -> dict:
proxy_config = settings_payload.get("proxy_config") or {}
proxy_enable = bool(proxy_config.get("proxy_enable", False))
allow_direct = bool(proxy_config.get("allow_direct", False))
refresh_status = str(runtime_state.get("proxy_last_refresh_status", "") or "").strip()
refresh_time = str(runtime_state.get("proxy_last_refresh_time", "") or "").strip()
source_count = len(proxy_config.get("proxy_urls", []))
source_stats = runtime_state.get("proxy_last_source_stats") or []
raw_items = int(runtime_state.get("proxy_last_refresh_total_items", 0) or 0)
validated = int(runtime_state.get("proxy_last_validated_count", 0) or 0)
available = int(runtime_state.get("proxy_last_available_count", available_proxy_count) or available_proxy_count)
source_ok_count = sum(1 for item in source_stats if str(item.get("status", "") or "").strip() == "ok")
supplier_empty = bool(source_stats) and source_ok_count == len(source_stats) and raw_items <= 0
if not proxy_enable:
return {
"state": "disabled",
"label": "未启用代理",
"detail": "当前使用直连模式，未启用代理池",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": refresh_status or "代理未启用",
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": False,
}
if available_proxy_count > 0:
detail = f"代理池当前可用 {available_proxy_count} 个代理，配置来源 {source_count} 个"
if refresh_status:
detail = f"{detail}；最近状态：{refresh_status}"
return {
"state": "healthy",
"label": "代理正常",
"detail": detail,
"direct_fallback_active": False,
"reason": "healthy",
"last_refresh_status": refresh_status,
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": False,
}
if allow_direct:
detail = "代理池当前无可用代理，已自动降级为直连继续执行"
reason = "no_available_proxy"
if supplier_empty:
reason = "supplier_empty_pool"
detail = "代理源最近都返回正常响应，但原始代理数为 0，当前判断为供应池为空；系统已自动降级为直连继续执行"
if refresh_status:
detail = f"{detail}；最近状态：{refresh_status}"
return {
"state": "degraded_direct",
"label": "降级直连",
"detail": detail,
"direct_fallback_active": True,
"reason": reason,
"last_refresh_status": refresh_status or "当前无可用代理",
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": supplier_empty,
}
detail = "代理池当前无可用代理，且未允许直连，检测链路会等待代理恢复"
reason = "no_available_proxy"
if supplier_empty:
reason = "supplier_empty_pool"
detail = "代理源最近都返回正常响应，但原始代理数为 0，当前判断为供应池为空；由于未允许直连，检测链路会等待代理恢复"
if refresh_status:
detail = f"{detail}；最近状态：{refresh_status}"
return {
"state": "blocked_no_proxy",
"label": "等待代理",
"detail": detail,
"direct_fallback_active": False,
"reason": reason,
"last_refresh_status": refresh_status or "当前无可用代理",
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": supplier_empty,
}
def get_detect_status() -> dict:
queries = {
"pending": "select count(*) from domains where detect_status = 0",
@@ -37,14 +297,65 @@ def get_detect_status() -> dict:
last_log_time = modified.isoformat()
worker_online = (datetime.now(timezone.utc) - modified).total_seconds() < 180
recent_lines = tail_lines("detect_worker.log", max_lines=80)
recent_proxy_warning = next(
(line for line in reversed(recent_lines) if "代理" in line or "Redis订阅失败" in line),
"",
)
recent_lines = tail_lines("detect_worker.log", max_lines=160)
runtime_state = _load_runtime_state()
runtime_started_at = runtime.get("latest_start_time", "") if 'runtime' in locals() else ""
recent_lines = _filter_lines_since(recent_lines, runtime_started_at)
available_proxy_count = _extract_available_proxy_count(recent_lines)
active_thread_snapshot = _extract_active_thread_snapshot(recent_lines)
if runtime_state:
available_proxy_count = int(runtime_state.get("available_proxy_count", available_proxy_count) or available_proxy_count)
active_thread_snapshot = {
"active": int(runtime_state.get("active_threads", active_thread_snapshot["active"]) or active_thread_snapshot["active"]),
"max": int(runtime_state.get("max_threads", active_thread_snapshot["max"]) or active_thread_snapshot["max"]),
}
progress_total = sum(progress.values())
progress_done = progress.get("completed", 0) + progress.get("blacklisted", 0) + progress.get("failed", 0)
progress_percent = round((progress_done / progress_total) * 100, 2) if progress_total > 0 else 0
runtime = detect_worker_runtime()
runtime_started_at = runtime.get("latest_start_time", "")
recent_lines = _filter_lines_since(recent_lines, runtime_started_at)
recent_proxy_warning = _normalize_recent_warning(runtime_state, recent_lines, available_proxy_count)
proxy_runtime = _build_proxy_runtime_snapshot(settings_payload, runtime_state, available_proxy_count)
thread_count_resolution = resolve_thread_count(settings_payload=settings_payload)
effective_thread_count = int(thread_count_resolution["effective_thread_count"])
runtime_settings = get_runtime_settings()
worker_online = worker_online or runtime.get("running", False)
if runtime_state.get("service_running") is True:
worker_online = True
if not runtime_state.get("detecting", False) and not progress.get("running", 0):
active_thread_snapshot = {"active": 0, "max": active_thread_snapshot["max"] or effective_thread_count}
settings_summary = {
"thread_count": effective_thread_count,
"thread_count_default": int(thread_count_resolution["default_thread_count"]),
"thread_count_source": str(thread_count_resolution["source"]),
"thread_count_override": thread_count_resolution["override_thread_count"],
"thread_count_node_code": str(thread_count_resolution["node_code"]),
"proxy_enable": settings_payload["proxy_config"].get("proxy_enable", False),
"allow_direct": settings_payload["proxy_config"].get("allow_direct", False),
"proxy_pool_count": len(settings_payload["proxy_config"].get("proxy_urls", [])),
}
active_job = get_active_detect_job_summary()
runtime_snapshot = {
**runtime,
"detecting": runtime_state.get("detecting", False),
"proxy_runtime_state": proxy_runtime["state"],
"proxy_runtime_label": proxy_runtime["label"],
"proxy_runtime_detail": proxy_runtime["detail"],
"proxy_direct_fallback_active": proxy_runtime["direct_fallback_active"],
"proxy_runtime_reason": proxy_runtime["reason"],
"proxy_supplier_empty": proxy_runtime["supplier_empty"],
}
runs = sync_detect_runs(runtime_snapshot, progress, settings_summary, active_job=active_job)
dependency_alerts = _extract_dependency_alerts(recent_lines)
append_detect_result_projection_if_changed(
detect={
"active_job": active_job,
"progress": progress,
"phase_label": runtime_state.get("phase", ""),
"phase_detail": runtime_state.get("detail", ""),
}
)
return {
"worker_online": worker_online,
@@ -54,12 +365,41 @@ def get_detect_status() -> dict:
"worker_process_count": runtime.get("process_count", 0),
"worker_latest_start_time": runtime.get("latest_start_time", ""),
"worker_runtime_message": runtime.get("message", ""),
"thread_count": settings_payload["thread_count"],
"runtime_state": runtime_state,
"phase_label": runtime_state.get("phase", ""),
"phase_detail": runtime_state.get("detail", ""),
"detecting": runtime_state.get("detecting", False),
"thread_count": effective_thread_count,
"thread_count_default": int(thread_count_resolution["default_thread_count"]),
"thread_count_source": str(thread_count_resolution["source"]),
"thread_count_override": thread_count_resolution["override_thread_count"],
"thread_count_node_code": str(thread_count_resolution["node_code"]),
"active_thread_count": active_thread_snapshot["active"],
"max_thread_count": active_thread_snapshot["max"] or effective_thread_count,
"proxy_enable": settings_payload["proxy_config"].get("proxy_enable", False),
"allow_direct": settings_payload["proxy_config"].get("allow_direct", False),
"proxy_pool_count": len(settings_payload["proxy_config"].get("proxy_urls", [])),
"available_proxy_count": 0,
"available_proxy_count": available_proxy_count,
"proxy_runtime_state": proxy_runtime["state"],
"proxy_runtime_label": proxy_runtime["label"],
"proxy_runtime_detail": proxy_runtime["detail"],
"proxy_direct_fallback_active": proxy_runtime["direct_fallback_active"],
"proxy_runtime_reason": proxy_runtime["reason"],
"proxy_supplier_empty": proxy_runtime["supplier_empty"],
"proxy_last_refresh_status": proxy_runtime["last_refresh_status"],
"proxy_last_refresh_time": proxy_runtime["last_refresh_time"],
"proxy_last_refresh_source_count": proxy_runtime["source_count"],
"proxy_last_refresh_total_items": proxy_runtime["raw_items"],
"proxy_last_validated_count": proxy_runtime["validated_count"],
"proxy_last_available_count": proxy_runtime["available_count"],
"proxy_source_stats": proxy_runtime["source_stats"],
"dependency_alerts": dependency_alerts,
"last_worker_log_time": last_log_time,
"progress": progress,
"progress_percent": progress_percent,
"recent_event": runtime_state.get("detail") or _recent_event(recent_lines),
"recent_warning": recent_proxy_warning,
"log_lines": recent_lines,
"runs": runs,
"active_job": active_job,
}

344
domain-api/app/services/domains_service.py
View File

@@ -7,7 +7,7 @@ from app.core.db import get_db
DETECT_STATUS_LABELS = {
0: "待检测",
1: "检测完成",
1: "检测通过",
2: "检测中",
3: "黑名单",
4: "检测失败",
@@ -40,6 +40,14 @@ REVIEW_STATUS_LABELS = {
3: "人工拒绝",
}
SOURCE_TYPE_LABELS = {
1: "聚名一口价",
2: "聚名过期删除",
6: "手工录入",
7: "TXT 导入",
9: "其它",
}
BEIAN_STATUS_LABELS = {
1: "未检测",
2: "有备案",
@@ -47,6 +55,103 @@ BEIAN_STATUS_LABELS = {
}
def _format_timestamp(value) -> str | None:
if not value:
return None
return value.isoformat(sep=" ", timespec="seconds")
def _json_status_to_text(value) -> str:
if isinstance(value, dict):
status = value.get("status")
else:
status = None
return "是" if status else "否"
def _json_state(value) -> str:
if isinstance(value, dict):
return str(value.get("state") or "").strip()
return ""
def _json_message(value) -> str:
if isinstance(value, dict):
return str(value.get("message") or "").strip()
return ""
def _normalize_step_detail(label: str, value) -> dict:
payload = value if isinstance(value, dict) else {}
state = str(payload.get("state") or "").strip()
return {
"label": label,
"state": state or ("passed" if bool(payload.get("status")) else ""),
"status": bool(payload.get("status")) if isinstance(payload.get("status"), bool) else None,
"message": str(payload.get("message") or "").strip(),
"checked_at": str(payload.get("checked_at") or "").strip(),
"step": str(payload.get("step") or "").strip(),
"raw": payload,
}
def _build_step_details(row: tuple) -> list[dict]:
return [
_normalize_step_detail("百度历史收录", row[16]),
_normalize_step_detail("百度Site收录", row[17]),
{
"label": "标题为中文",
"state": "passed" if bool(row[18]) else "",
"status": bool(row[18]),
"message": "标题含中文" if bool(row[18]) else "",
"checked_at": "",
"step": "中文标题",
"raw": row[18],
},
_normalize_step_detail("360 Site收录", row[19]),
_normalize_step_detail("Google Site收录", row[20]),
_normalize_step_detail("时光机", row[21]),
_normalize_step_detail("站长之家", row[22]),
_normalize_step_detail("爱站网", row[23]),
]
def _summarize_step_details(step_details: list[dict]) -> dict:
degraded = [item for item in step_details if item.get("state") == "degraded"]
failed = [item for item in step_details if item.get("state") == "failed"]
blacklisted = [item for item in step_details if item.get("state") == "blacklisted"]
return {
"degraded_count": len(degraded),
"failed_count": len(failed),
"blacklisted_count": len(blacklisted),
"has_degraded": bool(degraded),
"has_failed": bool(failed),
"has_blacklisted_step": bool(blacklisted),
"summary_text": (
f"降级 {len(degraded)} / 失败 {len(failed)} / 命中 {len(blacklisted)}"
if degraded or failed or blacklisted
else "步骤正常"
),
}
def _bool_to_text(value) -> str:
return "是" if bool(value) else "否"
def _normalize_detection_update(value) -> bool | None:
if value in (None, "", "skip"):
return None
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if text in {"是", "true", "1", "yes"}:
return True
if text in {"否", "false", "0", "no"}:
return False
raise ValueError("检测结果字段仅支持“是”或“否”")
def _build_domain_query_parts(filters: dict | None = None) -> tuple[str, str, list[object]]:
filters = filters or {}
conditions: list[str] = []
@@ -79,6 +184,12 @@ def _build_domain_query_parts(filters: dict | None = None) -> tuple[str, str, li
if filters.get("website_url"):
conditions.append("coalesce(d.website_url, '') ilike %s")
params.append(f"%{str(filters['website_url']).strip()}%")
if filters.get("company_type"):
conditions.append("coalesce(d.company_type, '') = %s")
params.append(str(filters["company_type"]).strip())
if filters.get("source_type") is not None:
conditions.append("d.source_type = %s")
params.append(int(filters["source_type"]))
if filters.get("backlink_gt_10"):
conditions.append("coalesce(dd.backlink_count_gt_10, false) = true")
@@ -102,6 +213,8 @@ def fetch_domains(
beian_year: int | None = None,
snapshot_year: str | None = None,
website_url: str | None = None,
company_type: str | None = None,
source_type: int | None = None,
backlink_gt_10: bool | None = None,
) -> dict:
offset = (page - 1) * page_size
@@ -115,6 +228,8 @@ def fetch_domains(
"beian_year": beian_year,
"snapshot_year": snapshot_year,
"website_url": website_url,
"company_type": company_type,
"source_type": source_type,
"backlink_gt_10": backlink_gt_10,
}
from_clause, where_clause, params = _build_domain_query_parts(filters)
@@ -138,7 +253,18 @@ def fetch_domains(
d.snapshot_years,
d.backlink_count,
d.detect_time,
coalesce(dd.backlink_count_gt_10, false) as backlink_gt_10
d.source_type,
coalesce(dd.backlink_count_gt_10, false) as backlink_gt_10,
d.expire_date,
d.company_type,
dd.baidu_history,
dd.baidu_site,
dd.is_chinese_title,
dd.qihu360_site,
dd.google_site,
dd.wayback_info,
dd.chinaz_info,
dd.aizhan_info
{from_clause}
{where_clause}
order by d.id desc
@@ -148,8 +274,11 @@ def fetch_domains(
)
rows = cur.fetchall()
items = [
{
items = []
for row in rows:
step_details = _build_step_details(row)
step_summary = _summarize_step_details(step_details)
items.append({
"id": row[0],
"domain": row[1],
"register_status": REGISTER_STATUS_LABELS.get(row[2], str(row[2])),
@@ -166,11 +295,19 @@ def fetch_domains(
"beian_year": row[8],
"snapshot_years": row[9] or "",
"backlink_count": row[10],
"detect_time": row[11].isoformat() if row[11] else None,
"backlink_gt_10": row[12],
}
for row in rows
]
"detect_time": _format_timestamp(row[11]),
"source_type": row[12],
"source_label": SOURCE_TYPE_LABELS.get(row[12], str(row[12])),
"backlink_gt_10": row[13],
"expire_date": _format_timestamp(row[14]),
"company_type": row[15] or "",
"baidu_history": _json_status_to_text(row[16]),
"baidu_site": _json_status_to_text(row[17]),
"is_chinese_title": _bool_to_text(row[18]),
"qihu360_site": _json_status_to_text(row[19]),
"google_site": _json_status_to_text(row[20]),
"step_summary": step_summary,
})
return {
"list": items,
"page": page,
@@ -180,12 +317,116 @@ def fetch_domains(
}
def fetch_domain_detail(domain_id: int) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
select
d.id,
d.domain,
d.register_status,
d.use_status,
d.detect_status,
d.review_status,
d.has_beian,
d.website_url,
d.beian_year,
d.snapshot_years,
d.backlink_count,
d.detect_time,
d.source_type,
coalesce(dd.backlink_count_gt_10, false) as backlink_gt_10,
d.expire_date,
d.company_type,
dd.baidu_history,
dd.baidu_site,
dd.is_chinese_title,
dd.qihu360_site,
dd.google_site,
dd.wayback_info,
dd.chinaz_info,
dd.aizhan_info,
dd.juziseo_info,
dd.jucha_info
from domains d
left join domain_detections dd on dd.domain_id = d.id
where d.id = %s
limit 1
""",
(int(domain_id),),
)
row = cur.fetchone()
if not row:
return None
step_details = [
_normalize_step_detail("百度历史收录", row[16]),
_normalize_step_detail("百度Site收录", row[17]),
{
"label": "标题为中文",
"state": "passed" if bool(row[18]) else "",
"status": bool(row[18]),
"message": "标题含中文" if bool(row[18]) else "",
"checked_at": "",
"step": "中文标题",
"raw": row[18],
},
_normalize_step_detail("360 Site收录", row[19]),
_normalize_step_detail("Google Site收录", row[20]),
_normalize_step_detail("时光机", row[21]),
_normalize_step_detail("站长之家", row[22]),
_normalize_step_detail("爱站网", row[23]),
_normalize_step_detail("桔子SEO", row[24]),
_normalize_step_detail("聚查", row[25]),
]
step_summary = _summarize_step_details(step_details)
return {
"id": row[0],
"domain": row[1],
"register_status": REGISTER_STATUS_LABELS.get(row[2], str(row[2])),
"register_status_code": row[2],
"use_status": USE_STATUS_LABELS.get(row[3], str(row[3])),
"use_status_code": row[3],
"detect_status": DETECT_STATUS_LABELS.get(row[4], str(row[4])),
"detect_status_code": row[4],
"review_status": REVIEW_STATUS_LABELS.get(row[5], str(row[5])),
"review_status_code": row[5],
"has_beian": BEIAN_STATUS_LABELS.get(row[6], str(row[6])),
"has_beian_code": row[6],
"website_url": row[7] or "",
"beian_year": row[8],
"snapshot_years": row[9] or "",
"backlink_count": row[10],
"detect_time": _format_timestamp(row[11]),
"source_type": row[12],
"source_label": SOURCE_TYPE_LABELS.get(row[12], str(row[12])),
"backlink_gt_10": row[13],
"expire_date": _format_timestamp(row[14]),
"company_type": row[15] or "",
"step_summary": step_summary,
"step_details": step_details,
"raw_detection": {
"baidu_history": row[16],
"baidu_site": row[17],
"is_chinese_title": row[18],
"qihu360_site": row[19],
"google_site": row[20],
"wayback_info": row[21],
"chinaz_info": row[22],
"aizhan_info": row[23],
"juziseo_info": row[24],
"jucha_info": row[25],
},
}
def domain_filter_options() -> dict:
return {
"register_status": [
{"label": label, "value": value}
for value, label in REGISTER_STATUS_LABELS.items()
if value in (2, 3, 4, 5, 6, 7, 8, 10)
if value in (0, 2, 3, 4, 5, 6, 7, 8, 9, 10)
],
"detect_status": [
{"label": label, "value": value}
@@ -204,6 +445,10 @@ def domain_filter_options() -> dict:
{"label": "有备案", "value": 2},
{"label": "无备案", "value": 3},
],
"source_type": [
{"label": label, "value": value}
for value, label in SOURCE_TYPE_LABELS.items()
],
"supports_backlink_gt_10": True,
"supports_txt_export": True,
"supports_excel_export": True,
@@ -225,11 +470,35 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict:
"detect_time",
"website_url",
"backlink_count",
"baidu_history",
"baidu_site",
"is_chinese_title",
"qihu360_site",
"google_site",
}
payload = {key: value for key, value in updates.items() if key in allowed_fields and value not in (None, "", "skip")}
if not payload:
raise ValueError("没有可更新的字段")
domain_fields = {
"review_status",
"expire_date",
"has_beian",
"beian_year",
"snapshot_years",
"company_type",
"detect_time",
"website_url",
"backlink_count",
}
detection_fields = {
"baidu_history",
"baidu_site",
"is_chinese_title",
"qihu360_site",
"google_site",
}
updated_count = 0
with get_db() as conn:
with conn.cursor() as cur:
@@ -238,6 +507,8 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict:
params: list[object] = []
for field, value in payload.items():
if field not in domain_fields:
continue
if field == "backlink_count":
set_parts.append("backlink_count = %s")
params.append(int(value))
@@ -245,11 +516,24 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict:
set_parts.append(f"{field} = %s")
params.append(value)
params.append(domain_id)
cur.execute(
f"update domains set {', '.join(set_parts)}, update_time = now() where id = %s",
tuple(params),
)
if set_parts:
params.append(domain_id)
cur.execute(
f"update domains set {', '.join(set_parts)}, update_time = now() where id = %s",
tuple(params),
)
detection_payload: dict[str, object] = {}
for field in detection_fields:
if field not in payload:
continue
normalized = _normalize_detection_update(payload[field])
if normalized is None:
continue
if field == "is_chinese_title":
detection_payload[field] = normalized
else:
detection_payload[field] = {"status": normalized}
if "backlink_count" in payload:
backlink_gt_10 = int(payload["backlink_count"]) > 10
@@ -268,6 +552,36 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict:
(domain_id, backlink_gt_10),
)
if detection_payload:
cur.execute("select id from domain_detections where domain_id = %s", (domain_id,))
existing_detection = cur.fetchone()
if existing_detection:
detection_set_parts: list[str] = []
detection_params: list[object] = []
for field, value in detection_payload.items():
detection_set_parts.append(f"{field} = %s")
detection_params.append(value)
detection_params.append(domain_id)
cur.execute(
f"""
update domain_detections
set {', '.join(detection_set_parts)}, update_time = now()
where domain_id = %s
""",
tuple(detection_params),
)
else:
insert_fields = ["domain_id", *detection_payload.keys(), "create_time", "update_time"]
placeholders = ["%s"] * (1 + len(detection_payload)) + ["now()", "now()"]
insert_params = [domain_id, *detection_payload.values()]
cur.execute(
f"""
insert into domain_detections ({', '.join(insert_fields)})
values ({', '.join(placeholders)})
""",
tuple(insert_params),
)
updated_count += 1
conn.commit()

47
domain-api/app/services/export_service.py
View File

@@ -23,8 +23,15 @@ EXPORT_HEADERS = [
("use_status", "使用状态"),
("detect_status", "检测状态"),
("review_status", "复核状态"),
("expire_date", "过期时间"),
("company_type", "单位性质"),
("has_beian", "备案状态"),
("website_url", "首页网址"),
("baidu_history", "百度历史收录"),
("baidu_site", "百度Site收录"),
("is_chinese_title", "标题为中文"),
("qihu360_site", "360 Site收录"),
("google_site", "Google Site收录"),
("beian_year", "备案年份"),
("snapshot_years", "快照年份"),
("backlink_count", "友链数"),
@@ -33,6 +40,18 @@ EXPORT_HEADERS = [
]
def _json_status_to_text(value) -> str:
if isinstance(value, dict):
status = value.get("status")
else:
status = None
return "是" if status else "否"
def _bool_to_text(value) -> str:
return "是" if bool(value) else "否"
def _normalize_payload(payload: dict) -> dict:
data = dict(payload or {})
data["page"] = int(data.get("page", 1) or 1)
@@ -68,8 +87,15 @@ def _query_export_rows(payload: dict) -> list[dict]:
d.use_status,
d.detect_status,
d.review_status,
d.expire_date,
d.company_type,
d.has_beian,
d.website_url,
dd.baidu_history,
dd.baidu_site,
dd.is_chinese_title,
dd.qihu360_site,
dd.google_site,
d.beian_year,
d.snapshot_years,
d.backlink_count,
@@ -93,13 +119,20 @@ def _query_export_rows(payload: dict) -> list[dict]:
"use_status": USE_STATUS_LABELS.get(row[2], str(row[2])),
"detect_status": DETECT_STATUS_LABELS.get(row[3], str(row[3])),
"review_status": REVIEW_STATUS_LABELS.get(row[4], str(row[4])),
"has_beian": BEIAN_STATUS_LABELS.get(row[5], str(row[5])),
"website_url": row[6] or "",
"beian_year": row[7] or "",
"snapshot_years": row[8] or "",
"backlink_count": row[9] or 0,
"backlink_gt_10": "是" if row[10] else "否",
"detect_time": row[11].isoformat(sep=" ", timespec="seconds") if row[11] else "",
"expire_date": row[5].isoformat(sep=" ", timespec="seconds") if row[5] else "",
"company_type": row[6] or "",
"has_beian": BEIAN_STATUS_LABELS.get(row[7], str(row[7])),
"website_url": row[8] or "",
"baidu_history": _json_status_to_text(row[9]),
"baidu_site": _json_status_to_text(row[10]),
"is_chinese_title": _bool_to_text(row[11]),
"qihu360_site": _json_status_to_text(row[12]),
"google_site": _json_status_to_text(row[13]),
"beian_year": row[14] or "",
"snapshot_years": row[15] or "",
"backlink_count": row[16] or 0,
"backlink_gt_10": "是" if row[17] else "否",
"detect_time": row[18].isoformat(sep=" ", timespec="seconds") if row[18] else "",
}
)
return result

95
domain-api/app/services/import_task_service.py
View File

@@ -10,6 +10,11 @@ from app.services.import_worker_service import import_domains_from_path
_IMPORT_TASK_LOCK = threading.Lock()
_SOURCE_TYPE_LABELS = {
6: "手工录入",
7: "TXT 导入",
9: "其它",
}
def _now() -> str:
@@ -24,6 +29,12 @@ def _save_tasks(tasks: list[dict]) -> None:
save_import_records(tasks)
def _append_log_locked(target: dict, message: str) -> None:
target.setdefault("logs", [])
target["logs"].append(f"[{_now()}] {message}")
target["logs"] = target["logs"][-200:]
def _update_task(task_id: str, **patch: object) -> dict | None:
with _IMPORT_TASK_LOCK:
tasks = load_import_records()
@@ -36,13 +47,73 @@ def _update_task(task_id: str, **patch: object) -> dict | None:
return dict(target)
def _update_task_with_log(task_id: str, log_message: str, **patch: object) -> dict | None:
with _IMPORT_TASK_LOCK:
tasks = load_import_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
return None
target.update(patch)
_append_log_locked(target, log_message)
target["updated_at"] = _now()
_save_tasks(tasks)
return dict(target)
def _phase_label(phase: str) -> str:
mapping = {
"queued": "排队中",
"reading": "读取文件中",
"normalizing": "清洗中",
"importing": "入库中",
"completed": "已完成",
"failed": "失败",
}
return mapping.get(phase, phase)
def _source_label(source_type: int) -> str:
return _SOURCE_TYPE_LABELS.get(int(source_type or 7), "未知")
def _run_import_task(task_id: str, file_path: str, source_type: int = 7) -> None:
_update_task(task_id, status="running", started_at=_now(), message="导入任务开始执行")
_update_task_with_log(
task_id,
f"导入任务开始执行，来源类型：{_source_label(source_type)}",
status="running",
started_at=_now(),
message=f"导入任务开始执行，来源类型：{_source_label(source_type)}",
phase="reading",
phase_label=_phase_label("reading"),
)
try:
result = import_domains_from_path(Path(file_path), source_type=source_type)
stats = result.get("stats", {})
_update_task(
path = Path(file_path)
_update_task_with_log(
task_id,
f"开始读取文件：{path.name}",
phase="reading",
phase_label=_phase_label("reading"),
)
raw_lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
total_lines = len(raw_lines)
non_empty = sum(1 for line in raw_lines if line.strip())
_update_task_with_log(
task_id,
f"文件读取完成，共 {total_lines} 行，非空 {non_empty} 行",
phase="normalizing",
phase_label=_phase_label("normalizing"),
message=f"文件读取完成，准备清洗 {non_empty} 条域名",
)
result = import_domains_from_path(path, source_type=source_type)
stats = result.get("stats", {})
_update_task_with_log(
task_id,
(
f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，"
f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}，"
f"来源类型 {result.get('source_label') or _source_label(source_type)}"
),
status="completed",
completed_at=_now(),
result=result,
@@ -50,13 +121,18 @@ def _run_import_task(task_id: str, file_path: str, source_type: int = 7) -> None
f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，"
f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}"
),
phase="completed",
phase_label=_phase_label("completed"),
)
except Exception as exc:
_update_task(
_update_task_with_log(
task_id,
f"导入失败：{exc}",
status="failed",
completed_at=_now(),
message=f"导入失败：{exc}",
phase="failed",
phase_label=_phase_label("failed"),
)
@@ -71,12 +147,16 @@ def create_import_task(content: bytes, filename: str, source_type: int = 7) -> d
"filename": safe_name,
"stored_path": str(target),
"source_type": source_type,
"source_label": _source_label(source_type),
"status": "queued",
"message": "文件已接收，等待处理",
"message": f"文件已接收，等待后台处理，来源类型：{_source_label(source_type)}",
"created_at": _now(),
"updated_at": _now(),
"started_at": "",
"completed_at": "",
"phase": "queued",
"phase_label": _phase_label("queued"),
"logs": [f"[{_now()}] 文件已接收，等待后台处理，来源类型：{_source_label(source_type)}"],
"result": None,
}
@@ -104,6 +184,9 @@ def retry_import_task(task_id: str) -> dict:
target["completed_at"] = ""
target["updated_at"] = _now()
target["result"] = None
target["phase"] = "queued"
target["phase_label"] = _phase_label("queued")
target["logs"] = [f"[{_now()}] 任务已重新加入队列，等待后台执行"]
_save_tasks(tasks)
stored_path = target["stored_path"]
source_type = int(target.get("source_type", 7))

7
domain-api/app/services/import_worker_service.py
View File

@@ -8,6 +8,11 @@ from app.core.files import import_root
DOMAIN_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(com|net)$", re.IGNORECASE)
SOURCE_TYPE_LABELS = {
6: "手工录入",
7: "TXT 导入",
9: "其它",
}
def normalize_domain(value: str) -> str | None:
@@ -92,6 +97,8 @@ def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict:
}
return {
"filename": file_path.name,
"source_type": source_type,
"source_label": SOURCE_TYPE_LABELS.get(int(source_type or 7), "未知"),
"stats": stats,
}

494
domain-api/app/services/juming_service.py Normal file
View File

@@ -0,0 +1,494 @@
from __future__ import annotations
import ast
import pickle
import re
import sys
from collections.abc import Callable
from datetime import date, datetime, timedelta
from pathlib import Path
from requests.cookies import RequestsCookieJar
from app.core.db import get_db
from app.core.redis_client import get_redis
from app.core.config import settings
from app.core.files import read_runtime_json, write_runtime_json
from app.services.import_worker_service import normalize_domain
DOMAIN_ROOT = Path(settings.domain_root)
if str(DOMAIN_ROOT) not in sys.path:
sys.path.insert(0, str(DOMAIN_ROOT))
from detect.juming import JM # type: ignore # noqa: E402
from detect.jucha import JC # type: ignore # noqa: E402
YKJ_DOMAIN_PATTERN = re.compile(r"<a class='yda1 ydz' ym='([^']*)'")
JUMING_COOKIE_FILE = DOMAIN_ROOT / "juming_cookies.pkl"
JUCHA_COOKIE_FILE = DOMAIN_ROOT / "jucha_cookies.pkl"
LEGACY_JUMING_COOKIE_FILES = [
Path.cwd() / "juming_cookies.pkl",
Path(__file__).resolve().parents[2] / "juming_cookies.pkl",
]
DELETE_LIST_SOURCE_TYPE = 2
FIXED_PRICE_SOURCE_TYPE = 1
JUMING_PREFERENCES_FILE = "juming_preferences.json"
class TaskStoppedError(RuntimeError):
pass
def _default_juming_preferences() -> dict:
return {
"mode": "delete_list",
"page_start": 1,
"page_size": 500,
"page_count": 1,
"crawl_date": date.today().isoformat(),
"auto_date": True,
}
def get_juming_preferences() -> dict:
defaults = _default_juming_preferences()
stored = read_runtime_json(JUMING_PREFERENCES_FILE, default={})
payload = {
"mode": str(stored.get("mode", defaults["mode"])) if stored else defaults["mode"],
"page_start": int(stored.get("page_start", defaults["page_start"])) if stored else defaults["page_start"],
"page_size": int(stored.get("page_size", defaults["page_size"])) if stored else defaults["page_size"],
"page_count": int(stored.get("page_count", defaults["page_count"])) if stored else defaults["page_count"],
"crawl_date": str(stored.get("crawl_date", defaults["crawl_date"])) if stored else defaults["crawl_date"],
"auto_date": bool(stored.get("auto_date", defaults["auto_date"])) if stored else defaults["auto_date"],
}
if payload["mode"] not in {"delete_list", "fixed_price"}:
payload["mode"] = defaults["mode"]
payload["page_start"] = max(payload["page_start"], 1)
payload["page_size"] = min(max(payload["page_size"], 1), 1000)
payload["page_count"] = min(max(payload["page_count"], 1), 20)
if not payload["crawl_date"]:
payload["crawl_date"] = defaults["crawl_date"]
return payload
def update_juming_preferences(payload: dict) -> dict:
current = get_juming_preferences()
next_payload = {
"mode": str(payload.get("mode", current["mode"]) or current["mode"]),
"page_start": int(payload.get("page_start", current["page_start"]) or current["page_start"]),
"page_size": int(payload.get("page_size", current["page_size"]) or current["page_size"]),
"page_count": int(payload.get("page_count", current["page_count"]) or current["page_count"]),
"crawl_date": str(payload.get("crawl_date", current["crawl_date"]) or current["crawl_date"]),
"auto_date": bool(payload.get("auto_date", current["auto_date"])),
}
if next_payload["mode"] not in {"delete_list", "fixed_price"}:
raise ValueError("无效的聚名采集类型")
next_payload["page_start"] = max(next_payload["page_start"], 1)
next_payload["page_size"] = min(max(next_payload["page_size"], 1), 1000)
next_payload["page_count"] = min(max(next_payload["page_count"], 1), 20)
write_runtime_json(JUMING_PREFERENCES_FILE, next_payload)
return next_payload
def _cookie_dict_to_jar(cookie_dict: dict[str, str]) -> RequestsCookieJar:
cookie_jar = RequestsCookieJar()
for name, value in cookie_dict.items():
cookie_jar.set(name, value)
return cookie_jar
def _emit_log(log: Callable[[str], None] | None, message: str) -> None:
if log:
log(message)
def _check_stop(should_stop: Callable[[], bool] | None) -> None:
if should_stop and should_stop():
raise TaskStoppedError("任务已停止")
def _cookie_jar_to_dict(cookie_jar: RequestsCookieJar) -> dict[str, str]:
return {str(cookie.name): str(cookie.value) for cookie in cookie_jar}
def _persist_juming_cookie(cookie_jar: RequestsCookieJar) -> None:
JUMING_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True)
with JUMING_COOKIE_FILE.open("wb") as handle:
pickle.dump(cookie_jar, handle)
try:
redis_client = get_redis()
redis_client.set("domain_tool:juming_cookies", str(_cookie_jar_to_dict(cookie_jar)))
except Exception:
pass
def _jucha_cookie_status() -> dict:
if JUCHA_COOKIE_FILE.exists():
return {
"cookie_ready": True,
"cookie_file": str(JUCHA_COOKIE_FILE),
}
return {
"cookie_ready": False,
"cookie_file": str(JUCHA_COOKIE_FILE),
}
def _load_juming_cookie() -> tuple[RequestsCookieJar | None, str]:
if JUMING_COOKIE_FILE.exists():
try:
with JUMING_COOKIE_FILE.open("rb") as handle:
loaded = pickle.load(handle)
if isinstance(loaded, RequestsCookieJar):
return loaded, "local"
if isinstance(loaded, dict):
return _cookie_dict_to_jar({str(k): str(v) for k, v in loaded.items()}), "local"
except Exception:
pass
for legacy_path in LEGACY_JUMING_COOKIE_FILES:
if not legacy_path.exists() or legacy_path == JUMING_COOKIE_FILE:
continue
try:
with legacy_path.open("rb") as handle:
loaded = pickle.load(handle)
if isinstance(loaded, RequestsCookieJar):
_persist_juming_cookie(loaded)
return loaded, f"migrated:{legacy_path}"
if isinstance(loaded, dict):
cookie_jar = _cookie_dict_to_jar({str(k): str(v) for k, v in loaded.items()})
_persist_juming_cookie(cookie_jar)
return cookie_jar, f"migrated:{legacy_path}"
except Exception:
continue
try:
redis_client = get_redis()
raw = redis_client.get("domain_tool:juming_cookies")
if raw:
parsed = ast.literal_eval(raw)
if isinstance(parsed, dict) and parsed:
cookie_jar = _cookie_dict_to_jar({str(k): str(v) for k, v in parsed.items()})
try:
with JUMING_COOKIE_FILE.open("wb") as handle:
pickle.dump(cookie_jar, handle)
except Exception:
pass
return cookie_jar, "redis"
except Exception:
pass
return None, "missing"
def get_juming_status() -> dict:
cookie_jar, storage = _load_juming_cookie()
status = {
"cookie_ready": cookie_jar is not None,
"cookie_storage": storage,
"cookie_file": str(JUMING_COOKIE_FILE),
"cookie_count": len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0,
"jucha": _jucha_cookie_status(),
"supported_modes": [
{"label": "聚名一口价", "value": "fixed_price", "source_type": FIXED_PRICE_SOURCE_TYPE},
{"label": "聚名过期删除", "value": "delete_list", "source_type": DELETE_LIST_SOURCE_TYPE},
],
"defaults": get_juming_preferences(),
}
status["linked_jucha"] = {
"attempted": False,
"ok": bool(status["jucha"]["cookie_ready"]),
"message": "聚查登录态已就绪" if status["jucha"]["cookie_ready"] else "尚未检测到聚查 Cookie",
}
return status
def login_juming(email: str, password: str) -> dict:
account = str(email or "").strip()
secret = str(password or "").strip()
if not account or not secret:
raise ValueError("请输入聚名账号和密码")
jm = JM()
jm.load_cookies()
login_result = jm.user_zh_p_login(account, secret)
if not login_result[0]:
raise ValueError(f"聚名登录失败: {login_result[1]}")
jm.save_cookies()
_persist_juming_cookie(jm.cookie)
linked_jucha = {
"attempted": True,
"ok": False,
"message": "未执行",
}
try:
jc = JC()
jc.load_juming_cookies()
linked_ok, linked_message = jc.auth_login()
linked_jucha["ok"] = bool(linked_ok)
linked_jucha["message"] = str(linked_message)
if linked_ok:
jc.save_cookies()
except Exception as exc:
linked_jucha["message"] = f"聚查联名登录失败: {exc}"
status = get_juming_status()
status["linked_jucha"] = linked_jucha
return status
def login_jucha_with_juming_cookie() -> dict:
if not JUMING_COOKIE_FILE.exists():
raise ValueError("请先完成聚名登录，当前未检测到聚名 Cookie")
jc = JC()
jc.load_juming_cookies()
linked_ok, linked_message = jc.auth_login()
if not linked_ok:
raise ValueError(f"聚查登录失败: {linked_message}")
jc.save_cookies()
status = get_juming_status()
status["linked_jucha"] = {
"attempted": True,
"ok": True,
"message": str(linked_message),
}
return status
def upload_juming_cookie(filename: str, content: bytes) -> dict:
name = Path(filename or "juming_cookies.pkl").name.lower()
if not (name.endswith(".pkl") or name.endswith(".pickle") or name.endswith(".json") or name.endswith(".txt")):
raise ValueError("仅支持上传 .pkl / .pickle / .json / .txt 格式的聚名 Cookie 文件")
cookie_jar: RequestsCookieJar | None = None
parse_error: str | None = None
if name.endswith(".pkl") or name.endswith(".pickle"):
try:
loaded = pickle.loads(content)
if isinstance(loaded, RequestsCookieJar):
cookie_jar = loaded
elif isinstance(loaded, dict):
cookie_jar = _cookie_dict_to_jar({str(k): str(v) for k, v in loaded.items()})
except Exception as exc:
parse_error = str(exc)
else:
try:
text = content.decode("utf-8")
parsed = ast.literal_eval(text)
if isinstance(parsed, dict):
cookie_jar = _cookie_dict_to_jar({str(k): str(v) for k, v in parsed.items()})
except Exception as exc:
parse_error = str(exc)
if cookie_jar is None:
raise ValueError(f"聚名 Cookie 文件解析失败: {parse_error or '内容不符合预期'}")
if not _cookie_jar_to_dict(cookie_jar):
raise ValueError("聚名 Cookie 文件为空，未检测到有效 Cookie")
_persist_juming_cookie(cookie_jar)
return {
"cookie_ready": True,
"cookie_storage": "upload",
"cookie_file": str(JUMING_COOKIE_FILE),
"cookie_count": len(_cookie_jar_to_dict(cookie_jar)),
}
def _insert_domains(
domains: list[str],
source_type: int,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> dict:
total = len(domains)
normalized_rows: list[tuple[str, str]] = []
invalid = 0
_emit_log(log, f"开始入库处理，共收到 {total} 个原始域名")
for value in domains:
_check_stop(should_stop)
normalized = normalize_domain(value)
if not normalized:
invalid += 1
continue
tld = normalized.rsplit(".", 1)[-1]
normalized_rows.append((normalized, tld))
existing_set: set[str] = set()
inserted = 0
with get_db() as conn:
with conn.cursor() as cur:
normalized_domains = [row[0] for row in normalized_rows]
if normalized_domains:
cur.execute("select domain from domains where domain = any(%s)", (normalized_domains,))
existing_set = {row[0] for row in cur.fetchall()}
if existing_set:
_emit_log(log, f"检测到 {len(existing_set)} 个已存在域名，将自动跳过")
inserted_since_commit = 0
for domain, tld in normalized_rows:
_check_stop(should_stop)
if domain in existing_set:
continue
cur.execute(
"""
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
) values (
%s, %s, %s, 0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
)
returning id
""",
(domain, tld, source_type),
)
domain_id = cur.fetchone()[0]
cur.execute(
"""
insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time)
values (%s, 1, 1, 5, 0, now(), now())
""",
(domain_id,),
)
inserted += 1
inserted_since_commit += 1
if inserted_since_commit >= 500:
conn.commit()
inserted_since_commit = 0
conn.commit()
valid = len(normalized_rows)
exists = len(existing_set)
_emit_log(log, f"入库完成：有效 {valid}，新增 {inserted}，已存在 {exists}，无效 {invalid}")
return {
"total": total,
"valid": valid,
"added": inserted,
"exists": exists,
"invalid": invalid,
"failed": max(valid - exists - inserted, 0),
}
def _crawl_fixed_price(
page_start: int,
page_size: int,
page_count: int,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> tuple[list[str], list[dict[str, int]]]:
cookie_jar, _ = _load_juming_cookie()
jm = JM()
jm.cookie = cookie_jar or RequestsCookieJar()
domains: list[str] = []
pages: list[dict[str, int]] = []
current_page = page_start
_emit_log(log, f"开始采集一口价域名：起始页 {page_start}，每页 {page_size}，最多 {page_count} 页")
for _ in range(page_count):
_check_stop(should_stop)
_emit_log(log, f"正在抓取第 {current_page} 页")
success, html = jm.ykj_get_list(page=current_page, page_size=page_size)
if not success:
raise RuntimeError(str(html))
page_domains = [item.strip() for item in YKJ_DOMAIN_PATTERN.findall(html) if item.strip()]
domains.extend(page_domains)
pages.append({"page": current_page, "count": len(page_domains)})
_emit_log(log, f"第 {current_page} 页抓取到 {len(page_domains)} 个域名，累计 {len(domains)} 个")
if len(page_domains) < page_size:
_emit_log(log, "当前页返回数量小于分页数量，判定已到末页，停止继续抓取")
break
current_page += 1
return domains, pages
def _crawl_delete_list(
crawl_date: str,
auto_date: bool,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> tuple[list[str], list[dict[str, int]]]:
cookie_jar, _ = _load_juming_cookie()
jm = JM()
jm.cookie = cookie_jar or RequestsCookieJar()
start_date = datetime.strptime(crawl_date, "%Y-%m-%d").date()
end_date = date.today() + timedelta(days=4)
current_date = start_date
domains: list[str] = []
dates: list[dict[str, int]] = []
_emit_log(log, f"开始采集删除列表：起始日期 {crawl_date}，自动追加日期 {'开启' if auto_date else '关闭'}")
while current_date <= end_date:
_check_stop(should_stop)
_emit_log(log, f"正在抓取 {current_date.isoformat()} 的删除列表")
domains_for_date = [item.strip() for item in jm.new_cha_del(current_date.isoformat()) if item.strip()]
domains.extend(domains_for_date)
dates.append({"date": current_date.isoformat(), "count": len(domains_for_date)})
_emit_log(log, f"{current_date.isoformat()} 抓取到 {len(domains_for_date)} 个域名，累计 {len(domains)} 个")
if not auto_date:
break
current_date += timedelta(days=1)
return domains, dates
def crawl_juming(
payload: dict,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> dict:
mode = str(payload.get("mode") or "delete_list").strip()
if mode not in {"fixed_price", "delete_list"}:
raise ValueError("仅支持 fixed_price 或 delete_list")
cookie_jar, storage = _load_juming_cookie()
if cookie_jar is None:
raise ValueError("未找到聚名 Cookie，请先在桌面版系统设置完成聚名登录，或将 Cookie 同步到服务器")
_emit_log(log, f"检测到聚名登录态，来源：{storage}")
_check_stop(should_stop)
if mode == "fixed_price":
page_start = max(int(payload.get("page_start") or 1), 1)
page_size = min(max(int(payload.get("page_size") or 500), 1), 1000)
page_count = min(max(int(payload.get("page_count") or 1), 1), 20)
domains, pages = _crawl_fixed_price(page_start, page_size, page_count, log=log, should_stop=should_stop)
stats = _insert_domains(domains, FIXED_PRICE_SOURCE_TYPE, log=log, should_stop=should_stop)
return {
"mode": mode,
"cookie_storage": storage,
"pages": pages,
"domains_found": len(domains),
"stats": stats,
"sample_domains": domains[:20],
}
crawl_date = str(payload.get("crawl_date") or date.today().isoformat())
auto_date = bool(payload.get("auto_date", True))
domains, dates = _crawl_delete_list(crawl_date, auto_date, log=log, should_stop=should_stop)
stats = _insert_domains(domains, DELETE_LIST_SOURCE_TYPE, log=log, should_stop=should_stop)
return {
"mode": mode,
"cookie_storage": storage,
"dates": dates,
"domains_found": len(domains),
"stats": stats,
"sample_domains": domains[:20],
}

206
domain-api/app/services/juming_task_service.py Normal file
View File

@@ -0,0 +1,206 @@
from __future__ import annotations
import threading
from datetime import datetime
from uuid import uuid4
from app.core.files import load_juming_records, save_juming_records
from app.services.juming_service import TaskStoppedError, crawl_juming
_JUMING_TASK_LOCK = threading.Lock()
_MAX_LOG_LINES = 400
def _now() -> str:
return datetime.now().isoformat(sep=" ", timespec="seconds")
def list_juming_tasks() -> list[dict]:
return load_juming_records()
def _save_tasks(tasks: list[dict]) -> None:
save_juming_records(tasks)
def _append_log(task_id: str, message: str) -> None:
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
return
logs = list(target.get("logs") or [])
logs.append(f"[{_now()}] {message}")
target["logs"] = logs[-_MAX_LOG_LINES:]
target["updated_at"] = _now()
_save_tasks(tasks)
def _update_task(task_id: str, **patch: object) -> dict | None:
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
return None
target.update(patch)
target["updated_at"] = _now()
_save_tasks(tasks)
return dict(target)
def _is_stop_requested(task_id: str) -> bool:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
return bool(target and target.get("cancel_requested"))
def _set_phase(task_id: str, phase: str, message: str | None = None) -> None:
phase_labels = {
"queued": "排队中",
"starting": "启动中",
"fetching": "抓取中",
"importing": "入库中",
"completed": "已完成",
"failed": "失败",
"stopping": "停止中",
"stopped": "已停止",
}
patch: dict[str, object] = {
"phase": phase,
"phase_label": phase_labels.get(phase, phase),
}
if message:
patch["message"] = message
_update_task(task_id, **patch)
def _log_and_track_phase(task_id: str, message: str) -> None:
if "开始采集" in message or "正在抓取" in message:
_set_phase(task_id, "fetching", message)
elif "开始入库处理" in message or "入库完成" in message or "已存在域名" in message:
_set_phase(task_id, "importing", message)
_append_log(task_id, message)
def _run_juming_task(task_id: str, payload: dict) -> None:
_update_task(task_id, status="running", started_at=_now(), message="聚名采集任务开始执行", cancel_requested=False)
_set_phase(task_id, "starting", "聚名采集任务开始执行")
_append_log(task_id, "任务已启动，正在准备读取聚名登录态")
try:
result = crawl_juming(
payload,
log=lambda message: _log_and_track_phase(task_id, str(message)),
should_stop=lambda: _is_stop_requested(task_id),
)
stats = result.get("stats", {})
_update_task(
task_id,
status="completed",
completed_at=_now(),
result=result,
cancel_requested=False,
message=(
f"采集完成：抓取 {result.get('domains_found', 0)} 个域名，"
f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}"
),
)
_set_phase(task_id, "completed")
_append_log(task_id, "任务执行完成")
except TaskStoppedError as exc:
_update_task(
task_id,
status="stopped",
completed_at=_now(),
cancel_requested=False,
message=str(exc),
)
_set_phase(task_id, "stopped")
_append_log(task_id, "任务已按请求停止")
except Exception as exc:
_update_task(
task_id,
status="failed",
completed_at=_now(),
cancel_requested=False,
message=f"采集失败：{exc}",
)
_set_phase(task_id, "failed")
_append_log(task_id, f"任务执行失败：{exc}")
def create_juming_task(payload: dict) -> dict:
task_id = uuid4().hex
mode = str(payload.get("mode") or "delete_list").strip() or "delete_list"
record = {
"task_id": task_id,
"mode": mode,
"payload": dict(payload or {}),
"status": "queued",
"phase": "queued",
"phase_label": "排队中",
"cancel_requested": False,
"message": "采集任务已创建，等待后台执行",
"created_at": _now(),
"updated_at": _now(),
"started_at": "",
"completed_at": "",
"result": None,
"logs": [f"[{_now()}] 已创建聚名采集任务，等待后台执行"],
}
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
tasks.insert(0, record)
_save_tasks(tasks)
worker = threading.Thread(target=_run_juming_task, args=(task_id, dict(payload or {})), daemon=True)
worker.start()
return record
def retry_juming_task(task_id: str) -> dict:
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
raise ValueError("聚名采集任务不存在")
if target.get("status") == "running":
raise ValueError("聚名采集任务正在运行，不能重复执行")
target["status"] = "queued"
target["phase"] = "queued"
target["phase_label"] = "排队中"
target["cancel_requested"] = False
target["message"] = "采集任务已重新加入队列"
target["started_at"] = ""
target["completed_at"] = ""
target["updated_at"] = _now()
target["result"] = None
target["logs"] = [f"[{_now()}] 已重新加入队列，等待后台执行"]
payload = dict(target.get("payload") or {})
record = dict(target)
_save_tasks(tasks)
worker = threading.Thread(target=_run_juming_task, args=(task_id, payload), daemon=True)
worker.start()
return record
def request_stop_juming_task(task_id: str) -> dict:
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
raise ValueError("聚名采集任务不存在")
if target.get("status") in {"completed", "failed", "stopped"}:
raise ValueError("当前任务已结束，无需停止")
target["cancel_requested"] = True
target["phase"] = "stopping"
target["phase_label"] = "停止中"
target["message"] = "已发送停止请求，等待当前步骤安全退出"
target["updated_at"] = _now()
logs = list(target.get("logs") or [])
logs.append(f"[{_now()}] 已收到停止请求，等待当前步骤安全退出")
target["logs"] = logs[-_MAX_LOG_LINES:]
_save_tasks(tasks)
return dict(target)

111
domain-api/app/services/juziseo_service.py Normal file
View File

@@ -0,0 +1,111 @@
from __future__ import annotations
import ast
import pickle
import sys
from pathlib import Path
from requests.cookies import RequestsCookieJar
from app.core.config import settings
from app.core.redis_client import get_redis
DOMAIN_ROOT = Path(settings.domain_root)
if str(DOMAIN_ROOT) not in sys.path:
sys.path.insert(0, str(DOMAIN_ROOT))
from detect.juziseo import Juziseo # type: ignore # noqa: E402
JUZISEO_COOKIE_FILE = DOMAIN_ROOT / "juziseo_cookies.pkl"
def _cookie_dict_to_jar(cookie_dict: dict[str, str]) -> RequestsCookieJar:
cookie_jar = RequestsCookieJar()
for name, value in cookie_dict.items():
cookie_jar.set(name, value)
return cookie_jar
def _cookie_jar_to_dict(cookie_jar: RequestsCookieJar) -> dict[str, str]:
return {str(cookie.name): str(cookie.value) for cookie in cookie_jar}
def _persist_juziseo_cookie(cookie_jar: RequestsCookieJar) -> None:
JUZISEO_COOKIE_FILE.parent.mkdir(parents=True, exist_ok=True)
with JUZISEO_COOKIE_FILE.open("wb") as handle:
pickle.dump(cookie_jar, handle)
try:
redis_client = get_redis()
redis_client.set("domain_tool:juziseo_cookies", str(_cookie_jar_to_dict(cookie_jar)))
except Exception:
pass
def _load_juziseo_cookie() -> tuple[RequestsCookieJar | None, str]:
if JUZISEO_COOKIE_FILE.exists():
try:
with JUZISEO_COOKIE_FILE.open("rb") as handle:
loaded = pickle.load(handle)
if isinstance(loaded, RequestsCookieJar):
return loaded, "local"
if isinstance(loaded, dict):
return _cookie_dict_to_jar({str(k): str(v) for k, v in loaded.items()}), "local"
except Exception:
pass
try:
redis_client = get_redis()
raw = redis_client.get("domain_tool:juziseo_cookies")
if raw:
parsed = ast.literal_eval(raw)
if isinstance(parsed, dict) and parsed:
cookie_jar = _cookie_dict_to_jar({str(k): str(v) for k, v in parsed.items()})
try:
with JUZISEO_COOKIE_FILE.open("wb") as handle:
pickle.dump(cookie_jar, handle)
except Exception:
pass
return cookie_jar, "redis"
except Exception:
pass
return None, "missing"
def get_juziseo_status() -> dict:
cookie_jar, storage = _load_juziseo_cookie()
return {
"cookie_ready": cookie_jar is not None,
"cookie_storage": storage,
"cookie_file": str(JUZISEO_COOKIE_FILE),
"cookie_count": len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0,
}
def login_juziseo(email: str, password: str) -> dict:
account = str(email or "").strip()
secret = str(password or "").strip()
if not account or not secret:
raise ValueError("请输入桔子SEO账号和密码")
juziseo = Juziseo()
juziseo.load_cookies(str(JUZISEO_COOKIE_FILE))
ok, message = juziseo.login(account, secret)
if not ok:
raise ValueError(f"桔子SEO登录失败: {message}")
try:
juziseo.save_cookies(str(JUZISEO_COOKIE_FILE))
except Exception:
pass
try:
_persist_juziseo_cookie(juziseo.cookie)
except Exception:
pass
status = get_juziseo_status()
status["message"] = str(message)
return status

50
domain-api/app/services/runtime_control_service.py
View File

@@ -5,8 +5,9 @@ import subprocess
from pathlib import Path
from app.core.config import settings
from app.services.sync_push_service import push_runtime_projection_now
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import start_worker, stop_worker
from app.services.worker_control_service import _run_systemctl, start_worker, stop_worker
def _workspace_root() -> Path:
@@ -23,7 +24,9 @@ def restart_api() -> tuple[bool, str]:
worker_mode = runtime.get("worker_mode", settings.worker_mode)
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "restart", api_service_name], timeout=30)
# When the API restarts itself under systemd, wait-free restart avoids
# blocking the HTTP request until uvicorn is torn down.
result = _run_systemctl(["restart", api_service_name], timeout=5, no_block=True)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "重启 Linux API 失败").strip()
return True, f"Linux API 重启命令已发送: {api_service_name}"
@@ -48,6 +51,35 @@ def restart_api() -> tuple[bool, str]:
return True, "API 重启命令已发送"
def _run_systemd_action(service_name: str, action: str, *, no_block: bool = False) -> tuple[bool, str]:
systemctl_command = []
if no_block:
systemctl_command.append("--no-block")
systemctl_command.extend([action, service_name])
result = _run_systemctl(systemctl_command, timeout=10 if no_block else 30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or f"{action} {service_name} 失败").strip()
return True, f"{service_name} {action} 命令已发送"
def start_sync_agent() -> tuple[bool, str]:
runtime = get_runtime_settings()
worker_mode = runtime.get("worker_mode", settings.worker_mode)
service_name = runtime.get("sync_agent_service_name", settings.sync_agent_service_name)
if worker_mode != "linux-systemd":
return False, "sync-agent 仅在 Linux systemd 多机部署中使用。"
return _run_systemd_action(service_name, "start")
def stop_sync_agent() -> tuple[bool, str]:
runtime = get_runtime_settings()
worker_mode = runtime.get("worker_mode", settings.worker_mode)
service_name = runtime.get("sync_agent_service_name", settings.sync_agent_service_name)
if worker_mode != "linux-systemd":
return False, "sync-agent 仅在 Linux systemd 多机部署中使用。"
return _run_systemd_action(service_name, "stop")
def runtime_action(action: str) -> tuple[bool, str, dict]:
if action == "start_worker":
ok, message = start_worker()
@@ -58,6 +90,20 @@ def runtime_action(action: str) -> tuple[bool, str, dict]:
if action == "restart_api":
ok, message = restart_api()
return ok, message, {"action": action, "poll_after_seconds": 4, "refresh_runtime": True}
if action == "start_sync_agent":
ok, message = start_sync_agent()
return ok, message, {"action": action, "poll_after_seconds": 2, "refresh_runtime": True}
if action == "stop_sync_agent":
ok, message = stop_sync_agent()
return ok, message, {"action": action, "poll_after_seconds": 2, "refresh_runtime": True}
if action == "push_sync":
ok, message, data = push_runtime_projection_now()
return ok, message, {
"action": action,
"poll_after_seconds": 2,
"refresh_runtime": True,
**(data or {}),
}
return False, f"不支持的运行时动作: {action}", {
"action": action,
"poll_after_seconds": 0,

1
domain-api/app/services/runtime_settings_service.py
View File

@@ -8,6 +8,7 @@ DEFAULT_RUNTIME_SETTINGS = {
"worker_mode": settings.worker_mode,
"worker_service_name": settings.worker_service_name,
"api_service_name": settings.api_service_name,
"sync_agent_service_name": settings.sync_agent_service_name,
}

271
domain-api/app/services/runtime_status_service.py
View File

@@ -5,9 +5,14 @@ from pathlib import Path
from app.core.config import settings
from app.core.db import get_db
from app.core.files import read_json
from app.core.redis_client import get_redis
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import detect_worker_runtime
from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime
def _runtime_log_path(filename: str) -> str:
@@ -15,10 +20,195 @@ def _runtime_log_path(filename: str) -> str:
return str(path)
def _domain_cookie_status(filename: str) -> tuple[bool, str]:
path = Path(settings.domain_root) / filename
return path.exists(), str(path)
def _bloom_filter_status() -> tuple[bool, str]:
try:
redis_client = get_redis()
modules = redis_client.execute_command("MODULE", "LIST")
for module in modules or []:
module_parts = module[::2]
module_values = module[1::2]
module_info = dict(zip(module_parts, module_values))
module_name = str(module_info.get("name", "")).lower()
if module_name in {"bf", "redisbloom"}:
return True, "RedisBloom 已安装"
return False, "Redis 未安装 RedisBloom，当前将使用普通缓存"
except Exception as exc:
return False, f"RedisBloom 检查失败: {exc}"
def _build_multi_region_readiness(
*,
cluster_snapshot: dict,
sync_summary: dict,
worker_runtime: dict,
sync_agent_runtime: dict,
) -> dict:
nodes = list(cluster_snapshot.get("nodes") or [])
summary = cluster_snapshot.get("summary") or {}
batch_summary = (sync_summary.get("detect_result_batches") or {})
batch_states = batch_summary.get("state_counts") or {}
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
mainland_control_nodes = [
node for node in nodes
if str(node.get("region") or "") == "mainland"
and str(node.get("role") or "") == "control"
and str(node.get("status") or "") in {"online", "busy"}
]
mainland_worker_nodes = [
node for node in nodes
if str(node.get("region") or "") == "mainland"
and str(node.get("role") or "") == "worker"
and str(node.get("status") or "") in {"online", "busy"}
]
blocking_issues: list[str] = []
warning_issues: list[str] = []
info_items: list[str] = []
if online_control_nodes <= 0:
blocking_issues.append("当前没有在线控制面节点，无法视为正式可用集群。")
if settings.node_region == "mainland" and settings.node_role == "control":
if not sync_agent_runtime.get("running", False):
blocking_issues.append("当前节点应承载 sync-agent，但服务未运行。")
if str(settings.sync_target_api_base_url or "").strip() == "":
blocking_issues.append("当前节点未配置 SYNC_TARGET_API_BASE_URL，无法向海外控制面推送。")
if not bool(settings.sync_push_enabled):
blocking_issues.append("当前节点未启用 SYNC_PUSH_ENABLED，结果同步不会自动执行。")
if settings.node_region == "overseas" and online_control_nodes > 0 and not mainland_control_nodes:
warning_issues.append("当前尚未观察到在线的大陆 controller 节点，后续自动结果同步仍未进入正式双地域态。")
if online_worker_nodes <= 0:
warning_issues.append("当前没有在线 Worker 节点，检测任务无法在多机状态下继续推进。")
offline_nodes = list(summary.get("offline_nodes") or [])
stale_nodes = list(summary.get("stale_nodes") or [])
if stale_nodes:
warning_issues.append(f"存在失活节点: {'、'.join(stale_nodes)}")
if offline_nodes:
warning_issues.append(f"存在离线节点: {'、'.join(offline_nodes)}")
failed_batches = int(batch_states.get("failed", 0) or 0)
projected_batches = int(batch_states.get("projected", 0) or 0)
pushing_batches = int(batch_states.get("pushing", 0) or 0)
synced_batches = int(batch_states.get("synced", 0) or 0)
if failed_batches > 0:
warning_issues.append(f"存在 {failed_batches} 个结果批次同步失败，需要检查 sync-agent 或目标接收面。")
if projected_batches > 0:
warning_issues.append(f"存在 {projected_batches} 个结果批次仍待推送。")
if pushing_batches > 0:
info_items.append(f"当前有 {pushing_batches} 个结果批次正在推送。")
if synced_batches > 0:
info_items.append(f"最近已接收 {synced_batches} 个结果批次。")
if worker_runtime.get("running", False):
info_items.append("当前节点本机 Worker 进程在线。")
if settings.node_region == "mainland" and settings.node_role == "control" and sync_agent_runtime.get("running", False):
info_items.append("当前节点本机 sync-agent 在线。")
if mainland_worker_nodes:
info_items.append(f"在线大陆 Worker {len(mainland_worker_nodes)} 台。")
if mainland_control_nodes:
info_items.append(f"在线大陆 controller {len(mainland_control_nodes)} 台。")
if blocking_issues:
status = "blocking"
summary_text = blocking_issues[0]
elif warning_issues:
status = "attention"
summary_text = warning_issues[0]
else:
status = "ready"
summary_text = "当前多机与跨地域骨架已进入可联调、可持续观察状态。"
return {
"status": status,
"ready": status == "ready",
"summary": summary_text,
"blocking_issues": blocking_issues,
"warnings": warning_issues,
"info": info_items,
"cluster": {
"online_control_nodes": online_control_nodes,
"online_worker_nodes": online_worker_nodes,
"mainland_control_nodes": len(mainland_control_nodes),
"mainland_worker_nodes": len(mainland_worker_nodes),
},
"sync": {
"enabled": bool(sync_summary.get("enabled", False)),
"source_region": sync_summary.get("source_region", ""),
"target_region": sync_summary.get("target_region", ""),
"projected_batches": projected_batches,
"pushing_batches": pushing_batches,
"failed_batches": failed_batches,
"synced_batches": synced_batches,
},
}
def get_runtime_status() -> dict:
runtime_settings = get_runtime_settings()
worker_runtime = detect_worker_runtime()
sync_agent_runtime = detect_sync_agent_runtime()
api_pid = os.getpid()
detect_snapshot = get_detect_status()
latest_run = (detect_snapshot.get("runs") or [None])[0] or {}
cluster_snapshot = get_cluster_snapshot()
queue_health = get_detect_queue_health(window_minutes=15)
effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0)
if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False):
effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1)
capacity_plan = get_detect_capacity_plan(
queue_health=queue_health,
online_worker_nodes=effective_online_worker_nodes,
target_finish_hours=6,
)
detect_payload = {
"phase_label": latest_run.get("phase_label", ""),
"phase_detail": latest_run.get("phase_detail", ""),
"recent_event": detect_snapshot.get("recent_event", ""),
"recent_warning": detect_snapshot.get("recent_warning", ""),
"progress_percent": detect_snapshot.get("progress_percent", 0),
"progress": detect_snapshot.get("progress", {}),
"active_thread_count": detect_snapshot.get("active_thread_count", 0),
"max_thread_count": detect_snapshot.get("max_thread_count", 0),
"available_proxy_count": detect_snapshot.get("available_proxy_count", 0),
"proxy_pool_count": detect_snapshot.get("proxy_pool_count", 0),
"proxy_runtime_label": detect_snapshot.get("proxy_runtime_label", ""),
"proxy_runtime_detail": detect_snapshot.get("proxy_runtime_detail", ""),
"proxy_runtime_reason": detect_snapshot.get("proxy_runtime_reason", ""),
"proxy_supplier_empty": detect_snapshot.get("proxy_supplier_empty", False),
"proxy_last_refresh_status": detect_snapshot.get("proxy_last_refresh_status", ""),
"proxy_last_refresh_time": detect_snapshot.get("proxy_last_refresh_time", ""),
"proxy_last_refresh_source_count": detect_snapshot.get("proxy_last_refresh_source_count", 0),
"proxy_last_refresh_total_items": detect_snapshot.get("proxy_last_refresh_total_items", 0),
"proxy_last_validated_count": detect_snapshot.get("proxy_last_validated_count", 0),
"proxy_last_available_count": detect_snapshot.get("proxy_last_available_count", 0),
"proxy_source_stats": detect_snapshot.get("proxy_source_stats", []),
"dependency_alerts": detect_snapshot.get("dependency_alerts", []),
"active_job": detect_snapshot.get("active_job"),
"runs_count": len(detect_snapshot.get("runs") or []),
"worker_online": worker_runtime.get("running", False),
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"queue_health": queue_health,
"capacity_plan": capacity_plan,
}
append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
sync_summary = get_sync_summary(record_limit=5)
readiness = _build_multi_region_readiness(
cluster_snapshot=cluster_snapshot,
sync_summary=sync_summary,
worker_runtime=worker_runtime,
sync_agent_runtime=sync_agent_runtime,
)
return {
"api": {
@@ -34,6 +224,11 @@ def get_runtime_status() -> dict:
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
},
"node": {
"code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
},
"worker": {
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
@@ -43,12 +238,26 @@ def get_runtime_status() -> dict:
"message": worker_runtime.get("message", ""),
"log_path": str(Path(settings.domain_root) / "detect_worker.log"),
},
"sync_agent": {
"mode": sync_agent_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("sync_agent_service_name", settings.sync_agent_service_name),
"running": sync_agent_runtime.get("running", False),
"process_count": sync_agent_runtime.get("process_count", 0),
"latest_start_time": sync_agent_runtime.get("latest_start_time", ""),
"message": sync_agent_runtime.get("message", ""),
"expected_on_this_node": settings.node_region == "mainland" and settings.node_role == "control",
},
"detect": detect_payload,
"cluster": cluster_snapshot,
"sync": sync_summary,
"readiness": readiness,
}
def get_runtime_preflight() -> dict:
runtime_settings = get_runtime_settings()
checks: list[dict[str, object]] = []
detect_options = read_json("detect_options.json", default={})
domain_root = Path(settings.domain_root)
checks.append(
@@ -75,10 +284,62 @@ def get_runtime_preflight() -> dict:
checks.append({"key": "redis", "label": "Redis", "ok": True, "message": f"{settings.redis_host}:{settings.redis_port}/{settings.redis_db}"})
except Exception as exc:
checks.append({"key": "redis", "label": "Redis", "ok": False, "message": str(exc)})
else:
bloom_ok, bloom_message = _bloom_filter_status()
checks.append(
{
"key": "redis_bloom",
"label": "RedisBloom",
"ok": True,
"message": bloom_message,
"level": "info" if bloom_ok else "warn",
"degraded": not bloom_ok,
}
)
worker_mode = runtime_settings.get("worker_mode", "windows-local")
checks.append({"key": "worker_mode", "label": "运行模式", "ok": True, "message": worker_mode})
jucha_enabled = bool(detect_options.get("detect_jucha"))
jucha_cookie_ok, jucha_cookie_path = _domain_cookie_status("jucha_cookies.pkl")
checks.append(
{
"key": "detect_jucha",
"label": "聚查检测",
"ok": True if not jucha_enabled else jucha_cookie_ok,
"message": "已启用" if jucha_enabled else "未启用",
}
)
checks.append(
{
"key": "jucha_cookie",
"label": "聚查 Cookie",
"ok": True if not jucha_enabled else jucha_cookie_ok,
"message": jucha_cookie_path if jucha_enabled else "未启用聚查检测，无需本地 Cookie",
"level": "info" if (not jucha_enabled or jucha_cookie_ok) else "warn",
}
)
juziseo_enabled = bool(detect_options.get("detect_juziseo"))
juziseo_cookie_ok, juziseo_cookie_path = _domain_cookie_status("juziseo_cookies.pkl")
checks.append(
{
"key": "detect_juziseo",
"label": "桔子SEO检测",
"ok": True if not juziseo_enabled else juziseo_cookie_ok,
"message": "已启用" if juziseo_enabled else "未启用",
}
)
checks.append(
{
"key": "juziseo_cookie",
"label": "桔子SEO Cookie",
"ok": True if not juziseo_enabled else juziseo_cookie_ok,
"message": juziseo_cookie_path if juziseo_enabled else "未启用桔子SEO检测，无需本地 Cookie",
"level": "info" if (not juziseo_enabled or juziseo_cookie_ok) else "warn",
}
)
if worker_mode == "linux-systemd":
checks.append(
{
@@ -96,6 +357,14 @@ def get_runtime_preflight() -> dict:
"message": runtime_settings.get("api_service_name", ""),
}
)
checks.append(
{
"key": "sync_agent_service_name",
"label": "Sync agent service 名",
"ok": bool(runtime_settings.get("sync_agent_service_name")),
"message": runtime_settings.get("sync_agent_service_name", ""),
}
)
else:
checks.append(
{

60
domain-api/app/services/sensitive_words_service.py Normal file
View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from app.core.db import get_db
def get_sensitive_words_payload() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
select word, category, priority
from sensitive_words
order by priority desc, word asc
"""
)
rows = cur.fetchall()
items = [
{
"word": row[0],
"category": row[1] or "default",
"priority": row[2] or 1,
}
for row in rows
]
return {
"items": items,
"text": "\n".join(item["word"] for item in items),
"total": len(items),
}
def save_sensitive_words_payload(payload: dict) -> dict:
raw_text = str(payload.get("text") or "")
words = []
seen: set[str] = set()
for line in raw_text.splitlines():
word = line.strip()
if not word or word in seen:
continue
seen.add(word)
words.append(word)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute("delete from sensitive_words")
if words:
cur.executemany(
"""
insert into sensitive_words (word, category, priority, create_time)
values (%s, 'default', 1, now())
""",
[(word,) for word in words],
)
conn.commit()
return {
"total": len(words),
"text": "\n".join(words),
}

165
domain-api/app/services/settings_service.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import json
from datetime import datetime
from app.core.config import settings as app_settings
from app.core.files import read_json, settings_backup_root, write_json
from app.core.redis_client import get_redis
from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings
@@ -12,6 +13,8 @@ REDIS_KEYS = {
"detect_options": "domain_tool:detect_options",
"proxy_config": "domain_tool:proxy_config",
"thread_count": "domain_tool:thread_count",
"node_thread_counts": "domain_tool:node_thread_counts",
"credentials": "domain_tool:credentials",
}
DETECT_OPTION_KEYS = {
@@ -26,10 +29,90 @@ DETECT_OPTION_KEYS = {
}
def _normalize_thread_count(value: object, *, field_name: str = "thread_count") -> int:
try:
thread_count = int(value)
except Exception as exc:
raise ValueError(f"{field_name} must be an integer") from exc
if thread_count < 1 or thread_count > 256:
raise ValueError(f"{field_name} out of range")
return thread_count
def _normalize_node_thread_counts(payload: object) -> dict[str, int]:
if payload in (None, ""):
return {}
if not isinstance(payload, dict):
raise ValueError("node_thread_counts must be an object")
normalized: dict[str, int] = {}
for raw_node_code, raw_thread_count in payload.items():
node_code = str(raw_node_code or "").strip()
if not node_code:
raise ValueError("node_thread_counts contains empty node code")
normalized[node_code] = _normalize_thread_count(raw_thread_count, field_name=f"node_thread_counts.{node_code}")
return normalized
def _load_thread_count_config() -> tuple[int, dict[str, int]]:
thread_count_payload = read_json("thread_count.json", default={"thread_count": "2"})
node_thread_counts_payload = read_json("node_thread_counts.json", default={})
try:
default_thread_count = _normalize_thread_count(thread_count_payload.get("thread_count", 2))
except ValueError:
default_thread_count = 2
try:
node_thread_counts = _normalize_node_thread_counts(node_thread_counts_payload)
except ValueError:
node_thread_counts = {}
redis_client = get_redis()
try:
if redis_thread_count := redis_client.get(REDIS_KEYS["thread_count"]):
try:
default_thread_count = _normalize_thread_count(redis_thread_count)
except ValueError:
pass
if redis_node_thread_counts := redis_client.get(REDIS_KEYS["node_thread_counts"]):
try:
node_thread_counts = _normalize_node_thread_counts(json.loads(redis_node_thread_counts))
except ValueError:
pass
except Exception:
pass
return default_thread_count, node_thread_counts
def resolve_thread_count(node_code: str | None = None, settings_payload: dict | None = None) -> dict:
payload = settings_payload or get_settings_payload()
default_thread_count = int(payload.get("thread_count", 2))
node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", {}))
normalized_node_code = str(node_code or app_settings.node_code or "").strip()
override_thread_count = None
source = "default"
effective_thread_count = default_thread_count
if normalized_node_code and normalized_node_code in node_thread_counts:
override_thread_count = node_thread_counts[normalized_node_code]
effective_thread_count = override_thread_count
source = "node_override"
return {
"node_code": normalized_node_code,
"default_thread_count": default_thread_count,
"effective_thread_count": effective_thread_count,
"override_thread_count": override_thread_count,
"source": source,
"node_thread_counts": node_thread_counts,
}
def get_settings_payload() -> dict:
detect_options = read_json("detect_options.json", default={})
proxy_config = read_json("proxy_config.json", default={})
thread_count = read_json("thread_count.json", default={"thread_count": "2"})
thread_count, node_thread_counts = _load_thread_count_config()
redis_client = get_redis()
try:
@@ -37,29 +120,84 @@ def get_settings_payload() -> dict:
detect_options = json.loads(redis_detect_options)
if redis_proxy_config := redis_client.get(REDIS_KEYS["proxy_config"]):
proxy_config = json.loads(redis_proxy_config)
if redis_thread_count := redis_client.get(REDIS_KEYS["thread_count"]):
thread_count = {"thread_count": str(redis_thread_count)}
except Exception:
pass
return {
"detect_options": detect_options,
"proxy_config": proxy_config,
"thread_count": int(thread_count.get("thread_count", 2)),
"thread_count": thread_count,
"node_thread_counts": node_thread_counts,
"current_node_code": app_settings.node_code,
"runtime_settings": get_runtime_settings(),
}
def get_credentials_payload() -> dict:
credentials = read_json(
"credentials.json",
default={
"juming": {"email": "", "password": ""},
"juziseo": {"email": "", "password": ""},
},
)
redis_client = get_redis()
try:
if redis_credentials := redis_client.get(REDIS_KEYS["credentials"]):
credentials = json.loads(redis_credentials)
except Exception:
pass
return {
"juming": {
"email": str(credentials.get("juming", {}).get("email", "")),
"password": str(credentials.get("juming", {}).get("password", "")),
},
"juziseo": {
"email": str(credentials.get("juziseo", {}).get("email", "")),
"password": str(credentials.get("juziseo", {}).get("password", "")),
},
}
def update_credentials_payload(payload: dict) -> dict:
current = get_credentials_payload()
credentials = {
"juming": {
"email": str(payload.get("juming", {}).get("email", current["juming"]["email"])),
"password": str(payload.get("juming", {}).get("password", current["juming"]["password"])),
},
"juziseo": {
"email": str(payload.get("juziseo", {}).get("email", current["juziseo"]["email"])),
"password": str(payload.get("juziseo", {}).get("password", current["juziseo"]["password"])),
},
}
write_json("credentials.json", credentials)
redis_client = get_redis()
try:
redis_client.set(REDIS_KEYS["credentials"], json.dumps(credentials, ensure_ascii=False))
redis_client.publish("domain_tool:credentials:update", json.dumps(credentials, ensure_ascii=False))
except Exception:
pass
return credentials
def update_settings_payload(payload: dict) -> dict:
current = get_settings_payload()
detect_options = payload.get("detect_options", current["detect_options"])
proxy_config = payload.get("proxy_config", current["proxy_config"])
thread_count = int(payload.get("thread_count", current["thread_count"]))
thread_count = _normalize_thread_count(payload.get("thread_count", current["thread_count"]))
node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", current.get("node_thread_counts", {})))
runtime_settings = update_runtime_settings(payload.get("runtime_settings", current["runtime_settings"]))
write_json("detect_options.json", detect_options)
write_json("proxy_config.json", proxy_config)
write_json("thread_count.json", {"thread_count": str(thread_count)})
write_json("node_thread_counts.json", node_thread_counts)
redis_client = get_redis()
try:
@@ -69,6 +207,8 @@ def update_settings_payload(payload: dict) -> dict:
redis_client.publish("domain_tool:proxy_config:update", json.dumps(proxy_config, ensure_ascii=False))
redis_client.set(REDIS_KEYS["thread_count"], thread_count)
redis_client.publish("domain_tool:thread_count:update", str(thread_count))
redis_client.set(REDIS_KEYS["node_thread_counts"], json.dumps(node_thread_counts, ensure_ascii=False))
redis_client.publish("domain_tool:node_thread_counts:update", json.dumps(node_thread_counts, ensure_ascii=False))
except Exception:
pass
@@ -76,6 +216,8 @@ def update_settings_payload(payload: dict) -> dict:
"detect_options": detect_options,
"proxy_config": proxy_config,
"thread_count": thread_count,
"node_thread_counts": node_thread_counts,
"current_node_code": app_settings.node_code,
"runtime_settings": runtime_settings,
}
@@ -101,12 +243,10 @@ def validate_settings_payload(payload: dict) -> None:
raise ValueError("invalid settings payload")
if "thread_count" in payload:
try:
thread_count = int(payload["thread_count"])
except Exception as exc:
raise ValueError("thread_count must be an integer") from exc
if thread_count < 1 or thread_count > 256:
raise ValueError("thread_count out of range")
_normalize_thread_count(payload["thread_count"])
if "node_thread_counts" in payload:
_normalize_node_thread_counts(payload["node_thread_counts"])
if "detect_options" in payload:
detect_options = payload["detect_options"]
@@ -135,6 +275,9 @@ def validate_settings_payload(payload: dict) -> None:
worker_mode = runtime_settings.get("worker_mode")
if worker_mode and worker_mode not in {"windows-local", "linux-systemd"}:
raise ValueError("worker_mode must be windows-local or linux-systemd")
for key in ("worker_service_name", "api_service_name", "sync_agent_service_name"):
if key in runtime_settings and runtime_settings[key] is not None and not str(runtime_settings[key]).strip():
raise ValueError(f"{key} must not be empty")
def backup_current_settings(reason: str = "manual") -> dict:

423
domain-api/app/services/sync_push_service.py Normal file
View File

@@ -0,0 +1,423 @@
from __future__ import annotations
import json
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.services.sync_record_service import _decode_json, _normalize_region
def _format_time(value: datetime | None) -> str:
return value.isoformat(sep=" ", timespec="seconds") if value else ""
def _ingest_url(base_url: str) -> str:
text = str(base_url or "").strip().rstrip("/")
if not text:
return ""
if text.endswith("/api/v1"):
return f"{text}/runtime/sync-ingest"
if text.endswith("/api/v1/runtime"):
return f"{text}/sync-ingest"
return f"{text}/api/v1/runtime/sync-ingest"
def _projection_ingest_type(sync_type: str) -> str:
if sync_type == "runtime_projection":
return "runtime_ingest"
if sync_type == "detect_result_projection":
return "detect_result_ingest"
return "sync_ingest"
def _load_latest_projection(sync_type: str) -> dict | None:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
FROM detect_sync_records
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(sync_type, source_region, target_region),
)
row = cur.fetchone()
if not row:
return None
return {
"id": row[0],
"source_region": row[1],
"target_region": row[2],
"status": row[3],
"payload": _decode_json(row[4]),
"created_at": row[5],
"updated_at": row[6],
}
def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200))))
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
FROM detect_sync_records
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
ORDER BY created_at ASC, id ASC
LIMIT %s
""",
(sync_type, source_region, target_region, safe_limit * 5),
)
rows = cur.fetchall()
selected: list[dict] = []
for row in rows:
projection = {
"id": row[0],
"source_region": row[1],
"target_region": row[2],
"status": row[3],
"payload": _decode_json(row[4]),
"created_at": row[5],
"updated_at": row[6],
}
latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type)
if latest_attempt and latest_attempt["status"] == "success":
continue
if latest_attempt and latest_attempt["status"] == "pending":
continue
if latest_attempt and latest_attempt["status"] == "failed":
last_created_at = latest_attempt.get("created_at")
if isinstance(last_created_at, datetime):
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
continue
selected.append(projection)
if len(selected) >= safe_limit:
break
return selected
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, status, payload_json, error_message, created_at, updated_at
FROM detect_sync_records
WHERE sync_type = 'runtime_push'
AND source_region = %s
AND target_region = %s
AND (payload_json->>'sync_type') = %s
AND (payload_json->>'source_record_id') = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(
_normalize_region(settings.sync_source_region, settings.node_region),
target_region,
sync_type,
str(int(source_record_id)),
),
)
row = cur.fetchone()
if not row:
return None
return {
"id": row[0],
"status": row[1],
"payload": _decode_json(row[2]),
"error_message": row[3] or "",
"created_at": row[4],
"updated_at": row[5],
}
def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int:
payload = {
"sync_type": sync_type,
"source_record_id": source_record["id"],
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
"ingest_url": ingest_url,
}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO detect_sync_records (
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id
""",
(
"runtime_push",
source_record["source_region"],
source_record["target_region"],
"pending",
json.dumps(payload, ensure_ascii=False),
"",
),
)
record_id = int(cur.fetchone()[0])
conn.commit()
return record_id
def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE detect_sync_records
SET status = %s,
payload_json = %s::jsonb,
error_message = %s,
updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""",
(
str(status or "").strip() or "pending",
json.dumps(payload or {}, ensure_ascii=False),
str(error_message or "").strip(),
int(record_id),
),
)
conn.commit()
def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
configured_token = str(settings.sync_shared_token or "").strip()
incoming_token = str(shared_token or "").strip()
if configured_token and incoming_token != configured_token:
return False, "同步 token 校验失败", {}
sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection"
ingest_sync_type = _projection_ingest_type(sync_type)
source_region = _normalize_region(payload.get("source_region"), "unknown")
source_record_id = int(payload.get("source_record_id") or 0)
projection_hash = str(payload.get("projection_hash") or "").strip()
projection = payload.get("projection") or {}
target_region = _normalize_region(settings.node_region, "overseas")
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM detect_sync_records
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
AND (payload_json->>'source_record_id') = %s
AND (payload_json->>'projection_hash') = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash),
)
existing = cur.fetchone()
if existing:
return True, "同步投影已存在，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
stored_payload = {
"sync_type": sync_type,
"source_record_id": source_record_id,
"projection_hash": projection_hash,
"projection": projection,
"received_at": _format_time(datetime.now()),
}
cur.execute(
"""
INSERT INTO detect_sync_records (
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id
""",
(
ingest_sync_type,
source_region,
target_region,
"received",
json.dumps(stored_payload, ensure_ascii=False),
"",
),
)
record_id = int(cur.fetchone()[0])
conn.commit()
return True, "同步投影接收成功", {"record_id": record_id, "deduplicated": False}
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
source_record = _load_latest_projection(sync_type)
if not source_record:
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
return _push_projection_record(source_record, sync_type, ingest_url)
def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200)))
if not pending_records:
latest = _load_latest_projection(sync_type)
if latest:
latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type)
if latest_attempt and latest_attempt["status"] == "success":
return True, "当前批次已全部同步完成", {
"action": "push_sync",
"sync_type": sync_type,
"deduplicated": True,
"batch_count": 0,
}
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0}
results: list[dict] = []
success_count = 0
for record in pending_records:
ok, message, data = _push_projection_record(record, sync_type, ingest_url)
results.append(
{
"source_record_id": record["id"],
"ok": ok,
"message": message,
"data": data,
}
)
if ok:
success_count += 1
overall_ok = success_count > 0
message = f"{sync_type} 批量推送完成，成功 {success_count}/{len(results)}"
return overall_ok, message, {
"action": "push_sync",
"sync_type": sync_type,
"batch_count": len(results),
"success_count": success_count,
"results": results,
}
def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type)
if latest_attempt and latest_attempt["status"] == "success":
return True, "该投影已推送，无需重复发送", {
"action": "push_sync",
"sync_type": sync_type,
"source_record_id": source_record["id"],
"deduplicated": True,
}
if latest_attempt and latest_attempt["status"] == "pending":
return True, "该投影已有同步推送进行中，暂不重复发送", {
"action": "push_sync",
"sync_type": sync_type,
"source_record_id": source_record["id"],
"deduplicated": True,
}
if latest_attempt and latest_attempt["status"] == "failed":
last_created_at = latest_attempt.get("created_at")
if isinstance(last_created_at, datetime):
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
return False, "最近一次同步推送刚失败，等待下个重试窗口", {
"action": "push_sync",
"sync_type": sync_type,
"source_record_id": source_record["id"],
"deduplicated": True,
}
attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type)
request_payload = {
"sync_type": sync_type,
"source_region": source_record["source_region"],
"source_record_id": source_record["id"],
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
"projection": (source_record.get("payload") or {}).get("projection", {}),
"created_at": _format_time(source_record.get("created_at")),
}
request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(
ingest_url,
data=request_body,
headers={
"Content-Type": "application/json",
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=15) as response:
raw = response.read().decode("utf-8")
data = json.loads(raw) if raw else {}
status_code = getattr(response, "status", 200)
except urllib.error.HTTPError as exc:
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
payload = {
"sync_type": sync_type,
"source_record_id": source_record["id"],
"projection_hash": request_payload["projection_hash"],
"ingest_url": ingest_url,
"http_status": getattr(exc, "code", 500),
"response_text": error_body[:1000],
}
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}")
return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
except Exception as exc:
payload = {
"sync_type": sync_type,
"source_record_id": source_record["id"],
"projection_hash": request_payload["projection_hash"],
"ingest_url": ingest_url,
}
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc))
return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
payload = {
"sync_type": sync_type,
"source_record_id": source_record["id"],
"projection_hash": request_payload["projection_hash"],
"ingest_url": ingest_url,
"http_status": status_code,
"response": data,
}
_update_push_attempt(attempt_id, status="success", payload=payload, error_message="")
return True, "投影推送成功", {
"action": "push_sync",
"sync_type": sync_type,
"attempt_id": attempt_id,
"source_record_id": source_record["id"],
"response": data,
}
def push_runtime_projection_now() -> tuple[bool, str, dict]:
if not settings.sync_push_enabled:
return False, "未启用同步推送", {"action": "push_sync"}
ingest_url = _ingest_url(settings.sync_target_api_base_url)
if not ingest_url:
return False, "未配置同步目标地址", {"action": "push_sync"}
results = []
ok, message, data = _push_projection_now("runtime_projection", ingest_url)
results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data})
ok, message, data = _push_projection_batch("detect_result_projection", ingest_url)
results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data})
success_count = sum(1 for item in results if item["ok"])
if success_count == 0:
return False, "当前没有成功推送的同步投影", {"action": "push_sync", "results": results}
return True, f"同步推送完成，成功 {success_count}/{len(results)}", {"action": "push_sync", "results": results}

592
domain-api/app/services/sync_record_service.py Normal file
View File

@@ -0,0 +1,592 @@
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
def _format_time(value: datetime | None) -> str:
return value.isoformat(sep=" ", timespec="seconds") if value else ""
def _decode_json(value: object) -> dict:
if isinstance(value, dict):
return value
if value in (None, ""):
return {}
try:
return json.loads(value)
except Exception:
return {}
def _normalize_region(value: str | None, fallback: str) -> str:
text = str(value or "").strip()
if not text or text == "unknown":
return str(fallback or "unknown").strip() or "unknown"
return text
def _should_append_runtime_projection(previous_payload: dict, current_projection: dict, previous_created_at: datetime | None) -> bool:
if not previous_payload:
return True
previous_projection = previous_payload.get("projection") or {}
if not previous_projection:
return True
keys_requiring_immediate_write = (
"worker_online",
"worker_mode",
"phase_label",
"phase_detail",
"proxy_runtime_label",
"proxy_runtime_reason",
)
for key in keys_requiring_immediate_write:
if previous_projection.get(key) != current_projection.get(key):
return True
previous_job = previous_projection.get("active_job") or {}
current_job = current_projection.get("active_job") or {}
for key in ("job_id", "job_code", "status"):
if previous_job.get(key) != current_job.get(key):
return True
previous_cluster = previous_projection.get("cluster_summary") or {}
current_cluster = current_projection.get("cluster_summary") or {}
for key in ("online_worker_nodes", "online_control_nodes", "busy_nodes", "stale_nodes", "offline_nodes"):
if previous_cluster.get(key) != current_cluster.get(key):
return True
previous_progress = previous_projection.get("progress") or {}
current_progress = current_projection.get("progress") or {}
failed_delta = abs(int(current_progress.get("failed", 0) or 0) - int(previous_progress.get("failed", 0) or 0))
blacklisted_delta = abs(int(current_progress.get("blacklisted", 0) or 0) - int(previous_progress.get("blacklisted", 0) or 0))
completed_delta = abs(int(current_progress.get("completed", 0) or 0) - int(previous_progress.get("completed", 0) or 0))
running_delta = abs(int(current_progress.get("running", 0) or 0) - int(previous_progress.get("running", 0) or 0))
if failed_delta > 0 or blacklisted_delta >= 10 or completed_delta >= 20 or running_delta >= 5:
return True
previous_alerts = previous_projection.get("dependency_alerts") or []
current_alerts = current_projection.get("dependency_alerts") or []
if previous_alerts != current_alerts:
return True
if not previous_created_at:
return True
now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now()
return now - previous_created_at >= timedelta(seconds=45)
def list_sync_records(limit: int = 20) -> list[dict]:
safe_limit = max(1, min(int(limit or 20), 200))
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
FROM detect_sync_records
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(safe_limit,),
)
rows = cur.fetchall()
return [
{
"id": row[0],
"sync_type": row[1],
"source_region": row[2],
"target_region": row[3],
"status": row[4],
"payload": _decode_json(row[5]),
"error_message": row[6] or "",
"created_at": _format_time(row[7]),
"updated_at": _format_time(row[8]),
}
for row in rows
]
def _latest_sync_record_by_source(
cur,
*,
sync_type: str,
source_region: str,
target_region: str,
source_record_id: int,
) -> dict | None:
cur.execute(
"""
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
FROM detect_sync_records
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
AND (payload_json->>'source_record_id') = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(sync_type, source_region, target_region, str(int(source_record_id))),
)
row = cur.fetchone()
if not row:
return None
return {
"id": row[0],
"sync_type": row[1],
"source_region": row[2],
"target_region": row[3],
"status": row[4],
"payload": _decode_json(row[5]),
"error_message": row[6] or "",
"created_at": _format_time(row[7]),
"updated_at": _format_time(row[8]),
}
def get_detect_result_sync_batches(limit: int = 5) -> dict:
safe_limit = max(1, min(int(limit or 5), 20))
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
batches: list[dict] = []
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, job_code, status, created_at, started_at, finished_at
FROM detect_jobs
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(safe_limit,),
)
jobs = cur.fetchall()
for row in jobs:
job_id = int(row[0])
job_code = str(row[1] or "")
job_status = str(row[2] or "")
created_at = _format_time(row[3])
started_at = _format_time(row[4])
finished_at = _format_time(row[5])
cur.execute(
"""
SELECT status, count(*)
FROM detect_job_items
WHERE job_id = %s
GROUP BY status
""",
(job_id,),
)
item_counts = {str(status or ""): int(count) for status, count in cur.fetchall()}
items_total = sum(item_counts.values())
items_terminal = int(item_counts.get("completed", 0)) + int(item_counts.get("blacklisted", 0)) + int(item_counts.get("failed", 0))
cur.execute(
"""
SELECT id, status, payload_json, created_at, updated_at
FROM detect_sync_records
WHERE sync_type = 'detect_result_projection'
AND source_region = %s
AND target_region = %s
AND (payload_json->'projection'->'job'->>'job_id') = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(source_region, target_region, str(job_id)),
)
projection_row = cur.fetchone()
projection = None
latest_push = None
latest_ingest = None
sync_state = "unsynced"
sync_message = "该任务还没有生成结果投影"
if projection_row:
projection_payload = _decode_json(projection_row[2])
projection = {
"id": int(projection_row[0]),
"status": projection_row[1],
"payload": projection_payload,
"created_at": _format_time(projection_row[3]),
"updated_at": _format_time(projection_row[4]),
}
latest_push = _latest_sync_record_by_source(
cur,
sync_type="runtime_push",
source_region=source_region,
target_region=target_region,
source_record_id=projection["id"],
)
latest_ingest = _latest_sync_record_by_source(
cur,
sync_type="detect_result_ingest",
source_region=source_region,
target_region=target_region,
source_record_id=projection["id"],
)
if latest_ingest:
sync_state = "synced"
sync_message = "最近一条结果投影已被目标地域接收"
elif latest_push and latest_push.get("status") == "success":
sync_state = "delivered"
sync_message = "结果投影已推送成功，等待目标侧回看接收记录"
elif latest_push and latest_push.get("status") == "pending":
sync_state = "pushing"
sync_message = "结果投影正在推送中"
elif latest_push and latest_push.get("status") == "failed":
sync_state = "failed"
sync_message = latest_push.get("error_message") or "最近一次结果投影推送失败"
else:
sync_state = "projected"
sync_message = "已生成结果投影，等待同步代理推送"
batches.append(
{
"job_id": job_id,
"job_code": job_code,
"job_status": job_status,
"created_at": created_at,
"started_at": started_at,
"finished_at": finished_at,
"items_total": items_total,
"items_terminal": items_terminal,
"items_pending": int(item_counts.get("pending", 0)),
"items_running": int(item_counts.get("running", 0)) + int(item_counts.get("claimed", 0)),
"items_failed": int(item_counts.get("failed", 0)),
"progress_percent": round((items_terminal / items_total) * 100, 2) if items_total else 0,
"sync_state": sync_state,
"sync_message": sync_message,
"projection": projection,
"latest_push": latest_push,
"latest_ingest": latest_ingest,
}
)
state_counts = {
"synced": 0,
"delivered": 0,
"pushing": 0,
"projected": 0,
"failed": 0,
"unsynced": 0,
}
for item in batches:
state = str(item.get("sync_state") or "unsynced")
state_counts[state] = state_counts.get(state, 0) + 1
return {
"source_region": source_region,
"target_region": target_region,
"jobs_total": len(batches),
"state_counts": state_counts,
"batches": batches,
}
def get_sync_summary(record_limit: int = 10) -> dict:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT status, count(*)
FROM detect_sync_records
GROUP BY status
"""
)
status_counts = {str(status or "unknown"): int(count) for status, count in cur.fetchall()}
cur.execute(
"""
SELECT sync_type, count(*)
FROM detect_sync_records
GROUP BY sync_type
"""
)
type_counts = {str(sync_type or "unknown"): int(count) for sync_type, count in cur.fetchall()}
cur.execute("SELECT count(*) FROM detect_sync_records")
total = int(cur.fetchone()[0] or 0)
cur.execute(
"""
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
FROM detect_sync_records
ORDER BY created_at DESC, id DESC
LIMIT 1
"""
)
latest = cur.fetchone()
latest_record = None
if latest:
latest_record = {
"id": latest[0],
"sync_type": latest[1],
"source_region": latest[2],
"target_region": latest[3],
"status": latest[4],
"payload": _decode_json(latest[5]),
"error_message": latest[6] or "",
"created_at": _format_time(latest[7]),
"updated_at": _format_time(latest[8]),
}
return {
"enabled": bool(settings.sync_push_enabled),
"source_region": source_region,
"target_region": target_region,
"target_api_base_url": settings.sync_target_api_base_url,
"batch_size": max(1, int(settings.sync_batch_size or 200)),
"poll_interval_seconds": max(5, int(settings.sync_poll_interval_seconds or 30)),
"records_total": total,
"status_counts": status_counts,
"type_counts": type_counts,
"latest_record": latest_record,
"detect_result_batches": get_detect_result_sync_batches(limit=min(5, record_limit)),
"recent_records": list_sync_records(limit=record_limit),
}
def append_sync_record(
*,
sync_type: str,
source_region: str,
target_region: str,
status: str,
payload: dict | None = None,
error_message: str = "",
) -> int:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO detect_sync_records (
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id
""",
(
str(sync_type or "").strip() or "unknown",
_normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)),
_normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")),
str(status or "").strip() or "pending",
json.dumps(payload or {}, ensure_ascii=False),
str(error_message or "").strip(),
),
)
record_id = int(cur.fetchone()[0])
conn.commit()
return record_id
def append_runtime_projection_if_changed(
*,
detect: dict,
cluster: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
active_job = detect.get("active_job") or {}
projection = {
"worker_online": bool(detect.get("worker_online", False)),
"worker_mode": detect.get("worker_mode", ""),
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
"progress": {
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
},
"active_job": {
"job_id": active_job.get("job_id"),
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
"progress_percent": active_job.get("progress_percent", 0),
"items_total": active_job.get("items_total", 0),
"items_terminal": active_job.get("items_terminal", 0),
"items_pending": active_job.get("items_pending", 0),
"items_running": active_job.get("items_running", 0),
"items_failed": active_job.get("items_failed", 0),
},
"cluster_summary": {
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
"online_control_nodes": int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0),
"busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []),
"stale_nodes": list((cluster.get("summary") or {}).get("stale_nodes") or []),
"offline_nodes": list((cluster.get("summary") or {}).get("offline_nodes") or []),
},
"dependency_alerts": [
{
"kind": item.get("kind", ""),
"title": item.get("title", ""),
"level": item.get("level", ""),
}
for item in (detect.get("dependency_alerts") or [])[:3]
],
}
payload = {
"projection": projection,
"projection_hash": hashlib.sha1(
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest(),
}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT payload_json, created_at
FROM detect_sync_records
WHERE sync_type = 'runtime_projection'
AND source_region = %s
AND target_region = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(normalized_source_region, normalized_target_region),
)
latest = cur.fetchone()
latest_payload = _decode_json(latest[0]) if latest else {}
latest_created_at = latest[1] if latest else None
if latest_payload.get("projection_hash") == payload["projection_hash"]:
return None
if not _should_append_runtime_projection(latest_payload, projection, latest_created_at):
return None
cur.execute(
"""
INSERT INTO detect_sync_records (
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id
""",
(
"runtime_projection",
normalized_source_region,
normalized_target_region,
"projected",
json.dumps(payload, ensure_ascii=False),
"",
),
)
record_id = int(cur.fetchone()[0])
conn.commit()
return record_id
def append_detect_result_projection_if_changed(
*,
detect: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
active_job = detect.get("active_job") or {}
if not active_job:
return None
latest_cycle_event = active_job.get("latest_cycle_event") or active_job.get("latest_event") or {}
projection = {
"job": {
"job_id": active_job.get("job_id"),
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
"progress_percent": active_job.get("progress_percent", 0),
"items_total": active_job.get("items_total", 0),
"items_pending": active_job.get("items_pending", 0),
"items_claimed": active_job.get("items_claimed", 0),
"items_running": active_job.get("items_running", 0),
"items_completed": active_job.get("items_completed", 0),
"items_blacklisted": active_job.get("items_blacklisted", 0),
"items_failed": active_job.get("items_failed", 0),
"items_terminal": active_job.get("items_terminal", 0),
"current_cycle_token": active_job.get("current_cycle_token", ""),
},
"latest_event": {
"node_code": latest_cycle_event.get("node_code", ""),
"event_type": latest_cycle_event.get("event_type", ""),
"message": latest_cycle_event.get("message", ""),
"created_at": latest_cycle_event.get("created_at", ""),
},
"queue": {
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
},
"phase": {
"label": detect.get("phase_label", ""),
"detail": detect.get("phase_detail", ""),
},
}
payload = {
"projection": projection,
"projection_hash": hashlib.sha1(
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest(),
}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT payload_json, created_at
FROM detect_sync_records
WHERE sync_type = 'detect_result_projection'
AND source_region = %s
AND target_region = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(normalized_source_region, normalized_target_region),
)
latest = cur.fetchone()
latest_payload = _decode_json(latest[0]) if latest else {}
latest_created_at = latest[1] if latest else None
if latest_payload.get("projection_hash") == payload["projection_hash"]:
return None
latest_projection = latest_payload.get("projection") or {}
latest_job = latest_projection.get("job") or {}
current_job = projection.get("job") or {}
latest_event = latest_projection.get("latest_event") or {}
current_event = projection.get("latest_event") or {}
if latest_job.get("status") == current_job.get("status") and latest_event == current_event and latest_created_at:
now = datetime.now(latest_created_at.tzinfo) if latest_created_at.tzinfo else datetime.now()
if now - latest_created_at < timedelta(seconds=30):
return None
cur.execute(
"""
INSERT INTO detect_sync_records (
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
RETURNING id
""",
(
"detect_result_projection",
normalized_source_region,
normalized_target_region,
"projected",
json.dumps(payload, ensure_ascii=False),
"",
),
)
record_id = int(cur.fetchone()[0])
conn.commit()
return record_id

124
domain-api/app/services/worker_control_service.py
View File

@@ -3,12 +3,18 @@ from __future__ import annotations
import json
import os
import subprocess
from datetime import datetime
from pathlib import Path
from app.core.config import settings
from app.core.redis_client import get_redis
from app.services.runtime_settings_service import get_runtime_settings
WORKER_CONTROL_CHANNEL = "domain_tool:worker_control"
WORKER_PENDING_COMMAND_KEY = "domain_tool:worker_pending_command"
def _domain_root() -> Path:
return Path(settings.domain_root)
@@ -30,6 +36,73 @@ def _run_shell(command: list[str], timeout: int = 20) -> subprocess.CompletedPro
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
def _run_systemctl(command: list[str], timeout: int = 20, require_sudo: bool = True) -> subprocess.CompletedProcess[str]:
systemctl_command = ["systemctl", *command]
if os.geteuid() == 0 or not require_sudo:
return _run_shell(systemctl_command, timeout=timeout)
return _run_shell(["sudo", "-n", *systemctl_command], timeout=timeout)
def _parse_systemd_timestamp(raw_timestamp: str) -> str:
raw_timestamp = (raw_timestamp or "").strip()
if not raw_timestamp:
return ""
try:
parsed = datetime.strptime(raw_timestamp, "%a %Y-%m-%d %H:%M:%S %Z")
return parsed.strftime("%Y-%m-%d %H:%M:%S")
except ValueError:
pass
try:
parsed = datetime.strptime(raw_timestamp.rsplit(" ", 1)[0], "%a %Y-%m-%d %H:%M:%S")
return parsed.strftime("%Y-%m-%d %H:%M:%S")
except ValueError:
return ""
def probe_systemd_service(service_name: str, *, mode: str = "linux-systemd") -> dict:
result = _run_systemctl(
[
"show",
service_name,
"--no-page",
"--property=ActiveState,SubState,MainPID,ExecMainStartTimestamp,ActiveEnterTimestamp",
],
require_sudo=False,
)
output = (result.stdout or result.stderr or "").strip()
if result.returncode != 0:
return {
"mode": mode,
"service_name": service_name,
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": output or f"systemd service {service_name} not available",
}
data: dict[str, str] = {}
for line in output.splitlines():
if "=" in line:
key, value = line.split("=", 1)
data[key] = value
main_pid = int(data.get("MainPID", "0") or 0)
active_state = data.get("ActiveState", "")
sub_state = data.get("SubState", "")
latest_start_time = ""
for raw_timestamp in (data.get("ExecMainStartTimestamp", ""), data.get("ActiveEnterTimestamp", "")):
latest_start_time = _parse_systemd_timestamp(raw_timestamp)
if latest_start_time:
break
return {
"mode": mode,
"service_name": service_name,
"running": active_state == "active",
"process_count": 1 if main_pid > 0 else 0,
"latest_start_time": latest_start_time,
"message": f"{active_state}/{sub_state}" if active_state else "",
}
def _windows_runtime() -> dict:
command = """
$targets = Get-CimInstance Win32_Process -Filter "name='python.exe'" |
@@ -83,32 +156,23 @@ def _windows_runtime() -> dict:
def _linux_runtime() -> dict:
runtime = _runtime_config()
service_name = runtime["worker_service_name"]
result = _run_shell(["systemctl", "show", service_name, "--no-page", "--property=ActiveState,SubState,MainPID"])
output = (result.stdout or result.stderr or "").strip()
if result.returncode != 0:
return probe_systemd_service(service_name, mode="linux-systemd")
def detect_sync_agent_runtime() -> dict:
runtime = _runtime_config()
worker_mode = runtime.get("worker_mode", settings.worker_mode)
service_name = runtime.get("sync_agent_service_name", settings.sync_agent_service_name)
if worker_mode != "linux-systemd":
return {
"mode": "linux-systemd",
"mode": worker_mode,
"service_name": service_name,
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": output or f"systemd service {service_name} not available",
"message": "sync-agent 仅在 Linux systemd 多机部署中使用",
}
data: dict[str, str] = {}
for line in output.splitlines():
if "=" in line:
key, value = line.split("=", 1)
data[key] = value
main_pid = int(data.get("MainPID", "0") or 0)
active_state = data.get("ActiveState", "")
sub_state = data.get("SubState", "")
return {
"mode": "linux-systemd",
"running": active_state == "active",
"process_count": 1 if main_pid > 0 else 0,
"latest_start_time": "",
"message": f"{active_state}/{sub_state}" if active_state else "",
}
return probe_systemd_service(service_name, mode="linux-systemd")
def detect_worker_runtime() -> dict:
@@ -132,7 +196,7 @@ def start_worker() -> tuple[bool, str]:
worker_mode = runtime["worker_mode"]
service_name = runtime["worker_service_name"]
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "start", service_name], timeout=30)
result = _run_systemctl(["start", service_name], timeout=30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "启动 Linux Worker 失败").strip()
return True, f"Linux Worker 启动命令已发送: {service_name}"
@@ -160,7 +224,7 @@ def stop_worker() -> tuple[bool, str]:
worker_mode = runtime["worker_mode"]
service_name = runtime["worker_service_name"]
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "stop", service_name], timeout=30)
result = _run_systemctl(["stop", service_name], timeout=30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "停止 Linux Worker 失败").strip()
return True, f"Linux Worker 停止命令已发送: {service_name}"
@@ -186,3 +250,17 @@ def stop_worker() -> tuple[bool, str]:
if "NO_PROCESS" in output:
return True, "当前没有运行中的检测端进程"
return True, output or "检测端已停止"
def send_worker_command(action: str, payload: dict | None = None) -> tuple[bool, str]:
try:
redis_client = get_redis()
command_payload = {"action": action}
if payload:
command_payload.update(payload)
serialized = json.dumps(command_payload, ensure_ascii=False)
redis_client.set(WORKER_PENDING_COMMAND_KEY, serialized, ex=120)
redis_client.publish(WORKER_CONTROL_CHANNEL, serialized)
return True, f"已发送 Worker 控制指令: {action}"
except Exception as exc:
return False, f"发送 Worker 控制指令失败: {exc}"