This commit is contained in:
Your Name
2026-04-16 13:05:07 +08:00
commit 32efff1670
99 changed files with 9974 additions and 0 deletions

View File

@@ -0,0 +1 @@

View File

@@ -0,0 +1,27 @@
from __future__ import annotations
from app.core.db import get_db
from app.services.runtime_status_service import get_runtime_status
def fetch_overview() -> dict:
queries = {
"domains_total": "select count(*) from domains",
"pending_total": "select count(*) from domains where detect_status = 0",
"completed_total": "select count(*) from domains where detect_status = 1",
"running_total": "select count(*) from domains where detect_status = 2",
"blacklist_total": "select count(*) from domains where detect_status = 3",
"failed_total": "select count(*) from domains where detect_status = 4",
"sensitive_words_total": "select count(*) from sensitive_words",
}
result: dict[str, int | str] = {}
with get_db() as conn:
with conn.cursor() as cur:
for key, query in queries.items():
cur.execute(query)
result[key] = cur.fetchone()[0]
runtime = get_runtime_status()
result["worker_status"] = "online" if runtime["worker"]["running"] else "offline"
result["api_status"] = "online"
result["worker_mode"] = runtime["worker"]["mode"]
return result

View File

@@ -0,0 +1,65 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from app.core.db import get_db
from app.core.files import tail_lines
from app.services.runtime_settings_service import get_runtime_settings
from app.services.settings_service import get_settings_payload
from app.services.worker_control_service import detect_worker_runtime
def get_detect_status() -> dict:
queries = {
"pending": "select count(*) from domains where detect_status = 0",
"completed": "select count(*) from domains where detect_status = 1",
"running": "select count(*) from domains where detect_status = 2",
"blacklisted": "select count(*) from domains where detect_status = 3",
"failed": "select count(*) from domains where detect_status = 4",
}
progress: dict[str, int] = {}
with get_db() as conn:
with conn.cursor() as cur:
for key, query in queries.items():
cur.execute(query)
progress[key] = cur.fetchone()[0]
settings_payload = get_settings_payload()
worker_log = Path(__file__).resolve().parents[3] / "domainCheck" / "detect_worker.log"
if not worker_log.exists():
worker_log = Path(__file__).resolve().parents[3] / "domainCheck" / "logs" / "detect_worker.log"
worker_online = False
last_log_time = None
if worker_log.exists():
modified = datetime.fromtimestamp(worker_log.stat().st_mtime, tz=timezone.utc)
last_log_time = modified.isoformat()
worker_online = (datetime.now(timezone.utc) - modified).total_seconds() < 180
recent_lines = tail_lines("detect_worker.log", max_lines=80)
recent_proxy_warning = next(
(line for line in reversed(recent_lines) if "代理" in line or "Redis订阅失败" in line),
"",
)
runtime = detect_worker_runtime()
runtime_settings = get_runtime_settings()
worker_online = worker_online or runtime.get("running", False)
return {
"worker_online": worker_online,
"worker_mode": runtime.get("mode", "windows-local"),
"worker_service_name": runtime_settings.get("worker_service_name", ""),
"api_service_name": runtime_settings.get("api_service_name", ""),
"worker_process_count": runtime.get("process_count", 0),
"worker_latest_start_time": runtime.get("latest_start_time", ""),
"worker_runtime_message": runtime.get("message", ""),
"thread_count": settings_payload["thread_count"],
"proxy_enable": settings_payload["proxy_config"].get("proxy_enable", False),
"allow_direct": settings_payload["proxy_config"].get("allow_direct", False),
"proxy_pool_count": len(settings_payload["proxy_config"].get("proxy_urls", [])),
"available_proxy_count": 0,
"last_worker_log_time": last_log_time,
"progress": progress,
"recent_warning": recent_proxy_warning,
}

View File

@@ -0,0 +1,277 @@
from __future__ import annotations
from math import ceil
from app.core.db import get_db
DETECT_STATUS_LABELS = {
0: "待检测",
1: "检测完成",
2: "检测中",
3: "黑名单",
4: "检测失败",
}
REGISTER_STATUS_LABELS = {
0: "待检测",
2: "可注册",
3: "已注册",
4: "宽限期",
5: "赎回期",
6: "删除期",
7: "clientHold",
8: "serverHold",
9: "状态未知",
10: "检测失败",
}
USE_STATUS_LABELS = {
0: "未使用",
1: "已经使用",
2: "已经卖出",
3: "已经预定",
}
REVIEW_STATUS_LABELS = {
0: "无需复核",
1: "待人工复核",
2: "人工通过",
3: "人工拒绝",
}
BEIAN_STATUS_LABELS = {
1: "未检测",
2: "有备案",
3: "无备案",
}
def _build_domain_query_parts(filters: dict | None = None) -> tuple[str, str, list[object]]:
filters = filters or {}
conditions: list[str] = []
params: list[object] = []
if filters.get("domain_keyword"):
conditions.append("d.domain ilike %s")
params.append(f"%{str(filters['domain_keyword']).strip()}%")
if filters.get("register_status") is not None:
conditions.append("d.register_status = %s")
params.append(filters["register_status"])
if filters.get("detect_status") is not None:
conditions.append("d.detect_status = %s")
params.append(filters["detect_status"])
if filters.get("use_status") is not None:
conditions.append("d.use_status = %s")
params.append(filters["use_status"])
if filters.get("review_status") is not None:
conditions.append("d.review_status = %s")
params.append(filters["review_status"])
if filters.get("has_beian") is not None:
conditions.append("d.has_beian = %s")
params.append(filters["has_beian"])
if filters.get("beian_year"):
conditions.append("d.beian_year = %s")
params.append(int(filters["beian_year"]))
if filters.get("snapshot_year"):
conditions.append("coalesce(d.snapshot_years, '') like %s")
params.append(f"%{str(filters['snapshot_year']).strip()}%")
if filters.get("website_url"):
conditions.append("coalesce(d.website_url, '') ilike %s")
params.append(f"%{str(filters['website_url']).strip()}%")
if filters.get("backlink_gt_10"):
conditions.append("coalesce(dd.backlink_count_gt_10, false) = true")
from_clause = """
from domains d
left join domain_detections dd on dd.domain_id = d.id
"""
where_clause = f"where {' and '.join(conditions)}" if conditions else ""
return from_clause, where_clause, params
def fetch_domains(
page: int = 1,
page_size: int = 20,
domain_keyword: str | None = None,
register_status: int | None = None,
detect_status: int | None = None,
has_beian: int | None = None,
use_status: int | None = None,
review_status: int | None = None,
beian_year: int | None = None,
snapshot_year: str | None = None,
website_url: str | None = None,
backlink_gt_10: bool | None = None,
) -> dict:
offset = (page - 1) * page_size
filters = {
"domain_keyword": domain_keyword,
"register_status": register_status,
"detect_status": detect_status,
"has_beian": has_beian,
"use_status": use_status,
"review_status": review_status,
"beian_year": beian_year,
"snapshot_year": snapshot_year,
"website_url": website_url,
"backlink_gt_10": backlink_gt_10,
}
from_clause, where_clause, params = _build_domain_query_parts(filters)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(f"select count(*) {from_clause} {where_clause}", tuple(params))
total = cur.fetchone()[0]
cur.execute(
f"""
select
d.id,
d.domain,
d.register_status,
d.use_status,
d.detect_status,
d.review_status,
d.has_beian,
d.website_url,
d.beian_year,
d.snapshot_years,
d.backlink_count,
d.detect_time,
coalesce(dd.backlink_count_gt_10, false) as backlink_gt_10
{from_clause}
{where_clause}
order by d.id desc
limit %s offset %s
""",
tuple(params + [page_size, offset]),
)
rows = cur.fetchall()
items = [
{
"id": row[0],
"domain": row[1],
"register_status": REGISTER_STATUS_LABELS.get(row[2], str(row[2])),
"register_status_code": row[2],
"use_status": USE_STATUS_LABELS.get(row[3], str(row[3])),
"use_status_code": row[3],
"detect_status": DETECT_STATUS_LABELS.get(row[4], str(row[4])),
"detect_status_code": row[4],
"review_status": REVIEW_STATUS_LABELS.get(row[5], str(row[5])),
"review_status_code": row[5],
"has_beian": BEIAN_STATUS_LABELS.get(row[6], str(row[6])),
"has_beian_code": row[6],
"website_url": row[7] or "",
"beian_year": row[8],
"snapshot_years": row[9] or "",
"backlink_count": row[10],
"detect_time": row[11].isoformat() if row[11] else None,
"backlink_gt_10": row[12],
}
for row in rows
]
return {
"list": items,
"page": page,
"page_size": page_size,
"total": total,
"pages": ceil(total / page_size) if page_size else 1,
}
def domain_filter_options() -> dict:
return {
"register_status": [
{"label": label, "value": value}
for value, label in REGISTER_STATUS_LABELS.items()
if value in (2, 3, 4, 5, 6, 7, 8, 10)
],
"detect_status": [
{"label": label, "value": value}
for value, label in DETECT_STATUS_LABELS.items()
],
"use_status": [
{"label": label, "value": value}
for value, label in USE_STATUS_LABELS.items()
],
"review_status": [
{"label": label, "value": value}
for value, label in REVIEW_STATUS_LABELS.items()
],
"has_beian": [
{"label": "未检测", "value": 1},
{"label": "有备案", "value": 2},
{"label": "无备案", "value": 3},
],
"supports_backlink_gt_10": True,
"supports_txt_export": True,
"supports_excel_export": True,
"supports_multi_page_export": True,
}
def batch_update_domains(domain_ids: list[int], updates: dict) -> dict:
if not domain_ids:
raise ValueError("未选择需要更新的域名")
allowed_fields = {
"review_status",
"expire_date",
"has_beian",
"beian_year",
"snapshot_years",
"company_type",
"detect_time",
"website_url",
"backlink_count",
}
payload = {key: value for key, value in updates.items() if key in allowed_fields and value not in (None, "", "skip")}
if not payload:
raise ValueError("没有可更新的字段")
updated_count = 0
with get_db() as conn:
with conn.cursor() as cur:
for domain_id in domain_ids:
set_parts: list[str] = []
params: list[object] = []
for field, value in payload.items():
if field == "backlink_count":
set_parts.append("backlink_count = %s")
params.append(int(value))
else:
set_parts.append(f"{field} = %s")
params.append(value)
params.append(domain_id)
cur.execute(
f"update domains set {', '.join(set_parts)}, update_time = now() where id = %s",
tuple(params),
)
if "backlink_count" in payload:
backlink_gt_10 = int(payload["backlink_count"]) > 10
cur.execute("select id from domain_detections where domain_id = %s", (domain_id,))
if cur.fetchone():
cur.execute(
"update domain_detections set backlink_count_gt_10 = %s, update_time = now() where domain_id = %s",
(backlink_gt_10, domain_id),
)
else:
cur.execute(
"""
insert into domain_detections (domain_id, backlink_count_gt_10, create_time, update_time)
values (%s, %s, now(), now())
""",
(domain_id, backlink_gt_10),
)
updated_count += 1
conn.commit()
return {
"updated_count": updated_count,
"fields": sorted(payload.keys()),
}

View File

@@ -0,0 +1,172 @@
from __future__ import annotations
import csv
from datetime import datetime
from openpyxl import Workbook
from app.core.db import get_db
from app.core.files import exports_root, load_export_records, save_export_record, timestamp_filename
from app.services.domains_service import (
BEIAN_STATUS_LABELS,
DETECT_STATUS_LABELS,
REGISTER_STATUS_LABELS,
REVIEW_STATUS_LABELS,
USE_STATUS_LABELS,
_build_domain_query_parts,
)
EXPORT_HEADERS = [
("domain", "域名"),
("register_status", "注册状态"),
("use_status", "使用状态"),
("detect_status", "检测状态"),
("review_status", "复核状态"),
("has_beian", "备案状态"),
("website_url", "首页网址"),
("beian_year", "备案年份"),
("snapshot_years", "快照年份"),
("backlink_count", "友链数"),
("backlink_gt_10", "友链>10"),
("detect_time", "检测时间"),
]
def _normalize_payload(payload: dict) -> dict:
data = dict(payload or {})
data["page"] = int(data.get("page", 1) or 1)
data["page_size"] = int(data.get("page_size", 100) or 100)
data["page_count"] = int(data.get("page_count", 1) or 1)
data["scope"] = data.get("scope", "page")
data["type"] = data.get("type", "txt")
return data
def _query_export_rows(payload: dict) -> list[dict]:
data = _normalize_payload(payload)
from_clause, where_clause, params = _build_domain_query_parts(data)
limit_offset = ""
if data["scope"] == "page":
offset = (data["page"] - 1) * data["page_size"]
limit_offset = " limit %s offset %s"
params.extend([data["page_size"], offset])
elif data["scope"] == "pages":
offset = (data["page"] - 1) * data["page_size"]
limit_value = data["page_size"] * max(data["page_count"], 1)
limit_offset = " limit %s offset %s"
params.extend([limit_value, offset])
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
f"""
select
d.domain,
d.register_status,
d.use_status,
d.detect_status,
d.review_status,
d.has_beian,
d.website_url,
d.beian_year,
d.snapshot_years,
d.backlink_count,
coalesce(dd.backlink_count_gt_10, false) as backlink_gt_10,
d.detect_time
{from_clause}
{where_clause}
order by d.id desc
{limit_offset}
""",
tuple(params),
)
rows = cur.fetchall()
result: list[dict] = []
for row in rows:
result.append(
{
"domain": row[0],
"register_status": REGISTER_STATUS_LABELS.get(row[1], str(row[1])),
"use_status": USE_STATUS_LABELS.get(row[2], str(row[2])),
"detect_status": DETECT_STATUS_LABELS.get(row[3], str(row[3])),
"review_status": REVIEW_STATUS_LABELS.get(row[4], str(row[4])),
"has_beian": BEIAN_STATUS_LABELS.get(row[5], str(row[5])),
"website_url": row[6] or "",
"beian_year": row[7] or "",
"snapshot_years": row[8] or "",
"backlink_count": row[9] or 0,
"backlink_gt_10": "" if row[10] else "",
"detect_time": row[11].isoformat(sep=" ", timespec="seconds") if row[11] else "",
}
)
return result
def _write_txt(path, rows: list[dict]) -> None:
with path.open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(f"{row['domain']}\n")
def _write_csv(path, rows: list[dict]) -> None:
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle)
writer.writerow([label for _, label in EXPORT_HEADERS])
for row in rows:
writer.writerow([row[key] for key, _ in EXPORT_HEADERS])
def _write_xlsx(path, rows: list[dict]) -> None:
workbook = Workbook()
sheet = workbook.active
sheet.title = "domains"
sheet.append([label for _, label in EXPORT_HEADERS])
for row in rows:
sheet.append([row[key] for key, _ in EXPORT_HEADERS])
workbook.save(path)
def create_export_file(payload: dict) -> dict:
data = _normalize_payload(payload)
rows = _query_export_rows(data)
ext = data["type"] if data["type"] in {"txt", "csv", "xlsx"} else "txt"
filename = timestamp_filename("domain_export", ext)
output_path = exports_root() / filename
if ext == "txt":
_write_txt(output_path, rows)
elif ext == "csv":
_write_csv(output_path, rows)
else:
_write_xlsx(output_path, rows)
created_at = datetime.fromtimestamp(output_path.stat().st_mtime)
record = {
"filename": filename,
"type": ext,
"scope": data["scope"],
"page": data["page"],
"page_size": data["page_size"],
"page_count": data["page_count"],
"count": len(rows),
"created_at": created_at.isoformat(sep=" ", timespec="seconds"),
"download_path": f"/api/v1/exports/download/{filename}",
}
save_export_record(record)
return record
def list_exports() -> list[dict]:
records = load_export_records()
normalized: list[dict] = []
for record in records:
item = dict(record)
created_at = item.get("created_at")
if isinstance(created_at, (int, float)):
item["created_at"] = datetime.fromtimestamp(created_at).isoformat(sep=" ", timespec="seconds")
normalized.append(item)
return normalized

View File

@@ -0,0 +1,114 @@
from __future__ import annotations
import threading
from datetime import datetime
from pathlib import Path
from uuid import uuid4
from app.core.files import import_root, load_import_records, save_import_records
from app.services.import_worker_service import import_domains_from_path
_IMPORT_TASK_LOCK = threading.Lock()
def _now() -> str:
return datetime.now().isoformat(sep=" ", timespec="seconds")
def list_import_tasks() -> list[dict]:
return load_import_records()
def _save_tasks(tasks: list[dict]) -> None:
save_import_records(tasks)
def _update_task(task_id: str, **patch: object) -> dict | None:
with _IMPORT_TASK_LOCK:
tasks = load_import_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
return None
target.update(patch)
target["updated_at"] = _now()
_save_tasks(tasks)
return dict(target)
def _run_import_task(task_id: str, file_path: str, source_type: int = 7) -> None:
_update_task(task_id, status="running", started_at=_now(), message="导入任务开始执行")
try:
result = import_domains_from_path(Path(file_path), source_type=source_type)
stats = result.get("stats", {})
_update_task(
task_id,
status="completed",
completed_at=_now(),
result=result,
message=(
f"导入完成:总数 {stats.get('total', 0)},有效 {stats.get('valid', 0)}"
f"新增 {stats.get('added', 0)},已存在 {stats.get('exists', 0)},无效 {stats.get('invalid', 0)}"
),
)
except Exception as exc:
_update_task(
task_id,
status="failed",
completed_at=_now(),
message=f"导入失败:{exc}",
)
def create_import_task(content: bytes, filename: str, source_type: int = 7) -> dict:
task_id = uuid4().hex
safe_name = Path(filename).name or "domains.txt"
target = import_root() / f"{task_id}_{safe_name}"
target.write_bytes(content)
record = {
"task_id": task_id,
"filename": safe_name,
"stored_path": str(target),
"source_type": source_type,
"status": "queued",
"message": "文件已接收,等待处理",
"created_at": _now(),
"updated_at": _now(),
"started_at": "",
"completed_at": "",
"result": None,
}
with _IMPORT_TASK_LOCK:
tasks = load_import_records()
tasks.insert(0, record)
_save_tasks(tasks)
worker = threading.Thread(target=_run_import_task, args=(task_id, str(target), source_type), daemon=True)
worker.start()
return record
def retry_import_task(task_id: str) -> dict:
with _IMPORT_TASK_LOCK:
tasks = load_import_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)
if not target:
raise ValueError("导入任务不存在")
if target.get("status") == "running":
raise ValueError("导入任务正在运行,不能重复执行")
target["status"] = "queued"
target["message"] = "任务已重新加入队列"
target["started_at"] = ""
target["completed_at"] = ""
target["updated_at"] = _now()
target["result"] = None
_save_tasks(tasks)
stored_path = target["stored_path"]
source_type = int(target.get("source_type", 7))
record = dict(target)
worker = threading.Thread(target=_run_import_task, args=(task_id, stored_path, source_type), daemon=True)
worker.start()
return record

View File

@@ -0,0 +1,103 @@
from __future__ import annotations
import re
from pathlib import Path
from app.core.db import get_db
from app.core.files import import_root
DOMAIN_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(com|net)$", re.IGNORECASE)
def normalize_domain(value: str) -> str | None:
candidate = value.strip().lower()
candidate = re.sub(r"^https?://", "", candidate)
candidate = candidate.split("/")[0].strip(".")
if candidate.startswith("www."):
candidate = candidate[4:]
if not DOMAIN_PATTERN.match(candidate):
return None
return candidate
def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict:
raw_lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines()
total = 0
normalized_rows: list[tuple[str, str]] = []
invalid = 0
for line in raw_lines:
line = line.strip()
if not line:
continue
total += 1
normalized = normalize_domain(line)
if not normalized:
invalid += 1
continue
tld = normalized.rsplit(".", 1)[-1]
normalized_rows.append((normalized, tld))
domains = [row[0] for row in normalized_rows]
existing_set: set[str] = set()
inserted = 0
with get_db() as conn:
with conn.cursor() as cur:
if domains:
cur.execute("select domain from domains where domain = any(%s)", (domains,))
existing_set = {row[0] for row in cur.fetchall()}
for domain, tld in normalized_rows:
if domain in existing_set:
continue
cur.execute(
"""
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
) values (
%s, %s, %s, 0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
)
returning id
""",
(domain, tld, source_type),
)
domain_id = cur.fetchone()[0]
cur.execute(
"""
insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time)
values (%s, 1, 1, 5, 0, now(), now())
""",
(domain_id,),
)
inserted += 1
conn.commit()
exists = len(existing_set)
valid = len(normalized_rows)
stats = {
"total": total,
"valid": valid,
"added": inserted,
"exists": exists,
"invalid": invalid,
"failed": max(valid - exists - inserted, 0),
}
return {
"filename": file_path.name,
"stats": stats,
}
def import_domains_from_upload(content: bytes, filename: str, source_type: int = 7) -> dict:
safe_name = Path(filename).name or "domains.txt"
target = import_root() / safe_name
target.write_bytes(content)
return import_domains_from_path(target, source_type=source_type)

View File

@@ -0,0 +1,25 @@
from __future__ import annotations
from app.core.db import get_db
from app.core.files import load_import_records
def get_import_summary() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute("select count(*) from domains")
domains_total = cur.fetchone()[0]
cur.execute("select count(*) from detect_tasks")
tasks_total = cur.fetchone()[0]
cur.execute("select max(create_time) from domains")
last_import_time = cur.fetchone()[0]
tasks = load_import_records()
running_tasks = sum(1 for item in tasks if item.get("status") in {"queued", "running"})
return {
"domains_total": domains_total,
"detect_tasks_total": tasks_total,
"last_import_time": last_import_time.isoformat() if last_import_time else None,
"import_task_total": len(tasks),
"running_import_tasks": running_tasks,
}

View File

@@ -0,0 +1,86 @@
from __future__ import annotations
import json
from datetime import datetime
from pathlib import Path
from zipfile import ZIP_DEFLATED, ZipFile
from app.core.files import runtime_root, tail_lines
from app.services.detect_service import get_detect_status
from app.services.runtime_status_service import get_runtime_status
from app.services.settings_service import get_settings_payload
def _tail_api_runtime_log(filename: str, max_lines: int = 80) -> list[str]:
path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename
if not path.exists():
return []
with path.open("r", encoding="utf-8", errors="replace") as handle:
return handle.read().splitlines()[-max_lines:]
def latest_logs() -> dict:
worker_lines = tail_lines("detect_worker.log", max_lines=80)
desktop_lines = tail_lines("logs/app.log", max_lines=60)
api_stdout_lines = _tail_api_runtime_log("domain-api.stdout.log", max_lines=60)
api_stderr_lines = _tail_api_runtime_log("domain-api.stderr.log", max_lines=60)
api_lines = api_stderr_lines + api_stdout_lines + desktop_lines
summary = "未发现显著异常"
level = "info"
if any("Redis订阅失败" in line for line in worker_lines):
summary = "检测端存在 Redis 订阅读超时重连,需要后续继续优化订阅策略。"
level = "warning"
elif any("无可用代理" in line for line in worker_lines):
summary = "代理池存在无可用代理情况,检测端当前可能回落直连或等待代理。"
level = "warning"
elif any("Traceback" in line or "ERROR:" in line for line in api_lines):
summary = "API 运行日志中发现异常堆栈,请优先检查 domain-api stderr 日志。"
level = "warning"
return {
"worker": worker_lines,
"api": api_lines,
"diagnostics": {
"summary": summary,
"level": level,
},
}
def build_diagnostic_bundle() -> tuple[Path, str]:
payload = latest_logs()
generated_at = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
bundle_dir = runtime_root() / "diagnostics"
bundle_dir.mkdir(parents=True, exist_ok=True)
zip_path = bundle_dir / f"diagnostic_bundle_{timestamp}.zip"
settings_payload = get_settings_payload()
detect_status = get_detect_status()
runtime_status = get_runtime_status()
with ZipFile(zip_path, "w", compression=ZIP_DEFLATED) as archive:
archive.writestr(
"summary.json",
json.dumps(
{
"generated_at": generated_at,
"diagnostics": payload["diagnostics"],
"runtime_status": runtime_status,
"detect_status": detect_status,
},
ensure_ascii=False,
indent=2,
default=str,
),
)
archive.writestr(
"settings_snapshot.json",
json.dumps(settings_payload, ensure_ascii=False, indent=2, default=str),
)
archive.writestr("logs/worker.log", "\n".join(payload["worker"]))
archive.writestr("logs/api.log", "\n".join(payload["api"]))
return zip_path, zip_path.name

View File

@@ -0,0 +1,65 @@
from __future__ import annotations
import os
import subprocess
from pathlib import Path
from app.core.config import settings
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import start_worker, stop_worker
def _workspace_root() -> Path:
return Path(settings.domain_root).parent
def _run_shell(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[str]:
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
def restart_api() -> tuple[bool, str]:
runtime = get_runtime_settings()
api_service_name = runtime.get("api_service_name", settings.api_service_name)
worker_mode = runtime.get("worker_mode", settings.worker_mode)
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "restart", api_service_name], timeout=30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "重启 Linux API 失败").strip()
return True, f"Linux API 重启命令已发送: {api_service_name}"
if os.name != "nt":
return False, "当前仅实现 Windows 本地 API 重启Linux 请将 worker_mode 设为 linux-systemd。"
workspace = _workspace_root()
stop_script = workspace / "stop_domain_api.ps1"
start_script = workspace / "start_domain_api.ps1"
if not stop_script.exists() or not start_script.exists():
return False, "未找到 API 启停脚本"
command = (
"Start-Process powershell "
"-ArgumentList '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', "
f"\"Start-Sleep -Seconds 2; & '{stop_script}'; Start-Sleep -Seconds 1; & '{start_script}'\""
)
result = _run_shell(["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command], timeout=20)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "重启 API 失败").strip()
return True, "API 重启命令已发送"
def runtime_action(action: str) -> tuple[bool, str, dict]:
if action == "start_worker":
ok, message = start_worker()
return ok, message, {"action": action, "poll_after_seconds": 2, "refresh_runtime": True}
if action == "stop_worker":
ok, message = stop_worker()
return ok, message, {"action": action, "poll_after_seconds": 2, "refresh_runtime": True}
if action == "restart_api":
ok, message = restart_api()
return ok, message, {"action": action, "poll_after_seconds": 4, "refresh_runtime": True}
return False, f"不支持的运行时动作: {action}", {
"action": action,
"poll_after_seconds": 0,
"refresh_runtime": False,
}

View File

@@ -0,0 +1,28 @@
from __future__ import annotations
from app.core.config import settings
from app.core.files import read_runtime_json, write_runtime_json
DEFAULT_RUNTIME_SETTINGS = {
"worker_mode": settings.worker_mode,
"worker_service_name": settings.worker_service_name,
"api_service_name": settings.api_service_name,
}
def get_runtime_settings() -> dict:
stored = read_runtime_json("runtime_settings.json", default={})
result = dict(DEFAULT_RUNTIME_SETTINGS)
result.update(stored or {})
return result
def update_runtime_settings(payload: dict) -> dict:
current = get_runtime_settings()
merged = dict(current)
for key in DEFAULT_RUNTIME_SETTINGS:
if key in payload and payload[key] is not None:
merged[key] = payload[key]
write_runtime_json("runtime_settings.json", merged)
return merged

View File

@@ -0,0 +1,113 @@
from __future__ import annotations
import os
from pathlib import Path
from app.core.config import settings
from app.core.db import get_db
from app.core.redis_client import get_redis
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import detect_worker_runtime
def _runtime_log_path(filename: str) -> str:
path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename
return str(path)
def get_runtime_status() -> dict:
runtime_settings = get_runtime_settings()
worker_runtime = detect_worker_runtime()
api_pid = os.getpid()
return {
"api": {
"service": "domain-api",
"version": "0.1.0",
"api_prefix": settings.api_prefix,
"pid": api_pid,
"host": settings.api_host,
"port": settings.api_port,
"mode": runtime_settings.get("worker_mode", "windows-local"),
"service_name": runtime_settings.get("api_service_name", settings.api_service_name),
"health_url": f"http://127.0.0.1:{settings.api_port}/health",
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
},
"worker": {
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
"running": worker_runtime.get("running", False),
"process_count": worker_runtime.get("process_count", 0),
"latest_start_time": worker_runtime.get("latest_start_time", ""),
"message": worker_runtime.get("message", ""),
"log_path": str(Path(settings.domain_root) / "detect_worker.log"),
},
}
def get_runtime_preflight() -> dict:
runtime_settings = get_runtime_settings()
checks: list[dict[str, object]] = []
domain_root = Path(settings.domain_root)
checks.append(
{
"key": "domain_root",
"label": "domainCheck 目录",
"ok": domain_root.exists(),
"message": str(domain_root),
}
)
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute("select 1")
cur.fetchone()
checks.append({"key": "database", "label": "PostgreSQL", "ok": True, "message": f"{settings.db_host}:{settings.db_port}/{settings.db_database}"})
except Exception as exc:
checks.append({"key": "database", "label": "PostgreSQL", "ok": False, "message": str(exc)})
try:
redis_client = get_redis()
redis_client.ping()
checks.append({"key": "redis", "label": "Redis", "ok": True, "message": f"{settings.redis_host}:{settings.redis_port}/{settings.redis_db}"})
except Exception as exc:
checks.append({"key": "redis", "label": "Redis", "ok": False, "message": str(exc)})
worker_mode = runtime_settings.get("worker_mode", "windows-local")
checks.append({"key": "worker_mode", "label": "运行模式", "ok": True, "message": worker_mode})
if worker_mode == "linux-systemd":
checks.append(
{
"key": "worker_service_name",
"label": "Worker service 名",
"ok": bool(runtime_settings.get("worker_service_name")),
"message": runtime_settings.get("worker_service_name", ""),
}
)
checks.append(
{
"key": "api_service_name",
"label": "API service 名",
"ok": bool(runtime_settings.get("api_service_name")),
"message": runtime_settings.get("api_service_name", ""),
}
)
else:
checks.append(
{
"key": "windows_scripts",
"label": "Windows 启停脚本",
"ok": (Path(settings.domain_root).parent / "start_domain_api.ps1").exists() and (Path(settings.domain_root).parent / "stop_domain_api.ps1").exists(),
"message": "start_domain_api.ps1 / stop_domain_api.ps1",
}
)
overall_ok = all(bool(item["ok"]) for item in checks)
return {
"ok": overall_ok,
"checks": checks,
}

View File

@@ -0,0 +1,172 @@
from __future__ import annotations
import json
from datetime import datetime
from app.core.files import read_json, settings_backup_root, write_json
from app.core.redis_client import get_redis
from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings
REDIS_KEYS = {
"detect_options": "domain_tool:detect_options",
"proxy_config": "domain_tool:proxy_config",
"thread_count": "domain_tool:thread_count",
}
DETECT_OPTION_KEYS = {
"detect_register",
"detect_wayback",
"detect_chinaz",
"detect_aizhan",
"detect_baidu_site",
"detect_360_site",
"detect_jucha",
"detect_juziseo",
}
def get_settings_payload() -> dict:
detect_options = read_json("detect_options.json", default={})
proxy_config = read_json("proxy_config.json", default={})
thread_count = read_json("thread_count.json", default={"thread_count": "2"})
redis_client = get_redis()
try:
if redis_detect_options := redis_client.get(REDIS_KEYS["detect_options"]):
detect_options = json.loads(redis_detect_options)
if redis_proxy_config := redis_client.get(REDIS_KEYS["proxy_config"]):
proxy_config = json.loads(redis_proxy_config)
if redis_thread_count := redis_client.get(REDIS_KEYS["thread_count"]):
thread_count = {"thread_count": str(redis_thread_count)}
except Exception:
pass
return {
"detect_options": detect_options,
"proxy_config": proxy_config,
"thread_count": int(thread_count.get("thread_count", 2)),
"runtime_settings": get_runtime_settings(),
}
def update_settings_payload(payload: dict) -> dict:
current = get_settings_payload()
detect_options = payload.get("detect_options", current["detect_options"])
proxy_config = payload.get("proxy_config", current["proxy_config"])
thread_count = int(payload.get("thread_count", current["thread_count"]))
runtime_settings = update_runtime_settings(payload.get("runtime_settings", current["runtime_settings"]))
write_json("detect_options.json", detect_options)
write_json("proxy_config.json", proxy_config)
write_json("thread_count.json", {"thread_count": str(thread_count)})
redis_client = get_redis()
try:
redis_client.set(REDIS_KEYS["detect_options"], json.dumps(detect_options, ensure_ascii=False))
redis_client.publish("domain_tool:detect_options:update", json.dumps(detect_options, ensure_ascii=False))
redis_client.set(REDIS_KEYS["proxy_config"], json.dumps(proxy_config, ensure_ascii=False))
redis_client.publish("domain_tool:proxy_config:update", json.dumps(proxy_config, ensure_ascii=False))
redis_client.set(REDIS_KEYS["thread_count"], thread_count)
redis_client.publish("domain_tool:thread_count:update", str(thread_count))
except Exception:
pass
return {
"detect_options": detect_options,
"proxy_config": proxy_config,
"thread_count": thread_count,
"runtime_settings": runtime_settings,
}
def export_settings_snapshot() -> dict:
return {
"schema_version": "1.0",
"exported_at": datetime.now().isoformat(),
"source": "domain-api",
"settings": get_settings_payload(),
}
def import_settings_snapshot(payload: dict) -> dict:
settings_payload = payload.get("settings", payload)
validate_settings_payload(settings_payload)
backup_current_settings("import")
return update_settings_payload(settings_payload)
def validate_settings_payload(payload: dict) -> None:
if not isinstance(payload, dict):
raise ValueError("invalid settings payload")
if "thread_count" in payload:
try:
thread_count = int(payload["thread_count"])
except Exception as exc:
raise ValueError("thread_count must be an integer") from exc
if thread_count < 1 or thread_count > 256:
raise ValueError("thread_count out of range")
if "detect_options" in payload:
detect_options = payload["detect_options"]
if not isinstance(detect_options, dict):
raise ValueError("detect_options must be an object")
detect_order = detect_options.get("detect_order", [])
if detect_order and not isinstance(detect_order, list):
raise ValueError("detect_order must be an array")
if isinstance(detect_order, list):
unknown_keys = [item for item in detect_order if item not in DETECT_OPTION_KEYS]
if unknown_keys:
raise ValueError(f"unknown detect option keys: {', '.join(unknown_keys)}")
if "proxy_config" in payload:
proxy_config = payload["proxy_config"]
if not isinstance(proxy_config, dict):
raise ValueError("proxy_config must be an object")
proxy_urls = proxy_config.get("proxy_urls", [])
if proxy_urls and not isinstance(proxy_urls, list):
raise ValueError("proxy_urls must be an array")
if "runtime_settings" in payload:
runtime_settings = payload["runtime_settings"]
if not isinstance(runtime_settings, dict):
raise ValueError("runtime_settings must be an object")
worker_mode = runtime_settings.get("worker_mode")
if worker_mode and worker_mode not in {"windows-local", "linux-systemd"}:
raise ValueError("worker_mode must be windows-local or linux-systemd")
def backup_current_settings(reason: str = "manual") -> dict:
snapshot = export_settings_snapshot()
snapshot["backup_reason"] = reason
filename = f"settings_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
path = settings_backup_root() / filename
path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
return {
"filename": filename,
"path": str(path),
}
def list_settings_backups(limit: int = 20) -> list[dict]:
root = settings_backup_root()
files = sorted(root.glob("settings_backup_*.json"), key=lambda item: item.stat().st_mtime, reverse=True)
result: list[dict] = []
for item in files[:limit]:
backup_reason = ""
try:
payload = json.loads(item.read_text(encoding="utf-8"))
backup_reason = str(payload.get("backup_reason", ""))
except Exception:
backup_reason = ""
result.append(
{
"filename": item.name,
"path": str(item),
"size": item.stat().st_size,
"modified_at": datetime.fromtimestamp(item.stat().st_mtime).isoformat(),
"backup_reason": backup_reason,
}
)
return result

View File

@@ -0,0 +1,188 @@
from __future__ import annotations
import json
import os
import subprocess
from pathlib import Path
from app.core.config import settings
from app.services.runtime_settings_service import get_runtime_settings
def _domain_root() -> Path:
return Path(settings.domain_root)
def _runtime_config() -> dict:
return get_runtime_settings()
def _run_powershell(command: str, timeout: int = 20) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command],
capture_output=True,
text=True,
timeout=timeout,
)
def _run_shell(command: list[str], timeout: int = 20) -> subprocess.CompletedProcess[str]:
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
def _windows_runtime() -> dict:
command = """
$targets = Get-CimInstance Win32_Process -Filter "name='python.exe'" |
Where-Object { $_.CommandLine -like '*detect_worker.py*' } |
Select-Object ProcessId, CommandLine
if (-not $targets) {
Write-Output '{"running":false,"process_count":0,"latest_start_time":"","mode":"windows-local"}'
exit 0
}
$latest = $null
foreach ($item in $targets) {
try {
$proc = Get-Process -Id $item.ProcessId -ErrorAction Stop
if (-not $latest -or $proc.StartTime -gt $latest.StartTime) {
$latest = $proc
}
} catch {}
}
$payload = @{
running = $true
process_count = @($targets).Count
latest_start_time = if ($latest) { $latest.StartTime.ToString('yyyy-MM-dd HH:mm:ss') } else { '' }
mode = 'windows-local'
} | ConvertTo-Json -Compress
Write-Output $payload
"""
result = _run_powershell(command)
output = (result.stdout or "").strip()
if result.returncode != 0 or not output:
return {
"mode": "windows-local",
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": (result.stderr or result.stdout or "worker runtime probe failed").strip(),
}
try:
payload = json.loads(output)
except json.JSONDecodeError:
return {
"mode": "windows-local",
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": output,
}
payload.setdefault("message", "")
return payload
def _linux_runtime() -> dict:
runtime = _runtime_config()
service_name = runtime["worker_service_name"]
result = _run_shell(["systemctl", "show", service_name, "--no-page", "--property=ActiveState,SubState,MainPID"])
output = (result.stdout or result.stderr or "").strip()
if result.returncode != 0:
return {
"mode": "linux-systemd",
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": output or f"systemd service {service_name} not available",
}
data: dict[str, str] = {}
for line in output.splitlines():
if "=" in line:
key, value = line.split("=", 1)
data[key] = value
main_pid = int(data.get("MainPID", "0") or 0)
active_state = data.get("ActiveState", "")
sub_state = data.get("SubState", "")
return {
"mode": "linux-systemd",
"running": active_state == "active",
"process_count": 1 if main_pid > 0 else 0,
"latest_start_time": "",
"message": f"{active_state}/{sub_state}" if active_state else "",
}
def detect_worker_runtime() -> dict:
runtime = _runtime_config()
worker_mode = runtime["worker_mode"]
if worker_mode == "linux-systemd":
return _linux_runtime()
if os.name == "nt":
return _windows_runtime()
return {
"mode": worker_mode,
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": f"unsupported worker_mode: {worker_mode}",
}
def start_worker() -> tuple[bool, str]:
runtime = _runtime_config()
worker_mode = runtime["worker_mode"]
service_name = runtime["worker_service_name"]
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "start", service_name], timeout=30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "启动 Linux Worker 失败").strip()
return True, f"Linux Worker 启动命令已发送: {service_name}"
if os.name != "nt":
return False, "当前仅实现 Windows 本地 Worker 启动Linux 请将 worker_mode 设为 linux-systemd。"
script_path = _domain_root() / "start_worker.ps1"
if not script_path.exists():
return False, f"未找到启动脚本: {script_path}"
command = (
"Start-Process powershell "
f"-ArgumentList '-ExecutionPolicy Bypass -File \"{script_path}\"' "
f"-WorkingDirectory '{_domain_root()}'"
)
result = _run_powershell(command)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "启动检测端失败").strip()
return True, "检测端启动命令已发送"
def stop_worker() -> tuple[bool, str]:
runtime = _runtime_config()
worker_mode = runtime["worker_mode"]
service_name = runtime["worker_service_name"]
if worker_mode == "linux-systemd":
result = _run_shell(["systemctl", "stop", service_name], timeout=30)
if result.returncode != 0:
return False, (result.stderr or result.stdout or "停止 Linux Worker 失败").strip()
return True, f"Linux Worker 停止命令已发送: {service_name}"
if os.name != "nt":
return False, "当前仅实现 Windows 本地 Worker 停止Linux 请将 worker_mode 设为 linux-systemd。"
command = """
$targets = Get-CimInstance Win32_Process -Filter "name='python.exe'" |
Where-Object { $_.CommandLine -like '*detect_worker.py*' } |
Select-Object -ExpandProperty ProcessId
if (-not $targets) {
Write-Output 'NO_PROCESS'
exit 0
}
$targets | ForEach-Object { Stop-Process -Id $_ -Force }
Write-Output ('STOPPED:' + (($targets | Measure-Object).Count))
"""
result = _run_powershell(command)
output = (result.stdout or result.stderr or "").strip()
if result.returncode != 0:
return False, output or "停止检测端失败"
if "NO_PROCESS" in output:
return True, "当前没有运行中的检测端进程"
return True, output or "检测端已停止"