434 lines
18 KiB
Python
434 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
|
|
from app.core.config import settings as app_settings
|
|
from app.core.files import read_json, settings_backup_root, write_json
|
|
from app.core.redis_client import get_redis
|
|
from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings
|
|
|
|
|
|
REDIS_KEYS = {
|
|
"detect_options": "domain_tool:detect_options",
|
|
"proxy_config": "domain_tool:proxy_config",
|
|
"thread_count": "domain_tool:thread_count",
|
|
"node_thread_counts": "domain_tool:node_thread_counts",
|
|
"process_count": "domain_tool:process_count",
|
|
"node_process_counts": "domain_tool:node_process_counts",
|
|
"credentials": "domain_tool:credentials",
|
|
"runtime_settings": "domain_tool:runtime_settings",
|
|
}
|
|
|
|
DETECT_OPTION_KEYS = {
|
|
"detect_register",
|
|
"detect_wayback",
|
|
"detect_chinaz",
|
|
"detect_aizhan",
|
|
"detect_baidu_site",
|
|
"detect_360_site",
|
|
"detect_jucha",
|
|
"detect_juziseo",
|
|
}
|
|
|
|
|
|
def _normalize_thread_count(value: object, *, field_name: str = "thread_count") -> int:
|
|
try:
|
|
thread_count = int(value)
|
|
except Exception as exc:
|
|
raise ValueError(f"{field_name} must be an integer") from exc
|
|
if thread_count < 1:
|
|
raise ValueError(f"{field_name} must be >= 1")
|
|
return thread_count
|
|
|
|
|
|
def _normalize_process_count(value: object, *, field_name: str = "process_count") -> int:
|
|
try:
|
|
process_count = int(value)
|
|
except Exception as exc:
|
|
raise ValueError(f"{field_name} must be an integer") from exc
|
|
if process_count < 1:
|
|
raise ValueError(f"{field_name} must be >= 1")
|
|
return process_count
|
|
|
|
|
|
def _normalize_node_thread_counts(payload: object) -> dict[str, int]:
|
|
if payload in (None, ""):
|
|
return {}
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("node_thread_counts must be an object")
|
|
|
|
normalized: dict[str, int] = {}
|
|
for raw_node_code, raw_thread_count in payload.items():
|
|
node_code = str(raw_node_code or "").strip()
|
|
if not node_code:
|
|
raise ValueError("node_thread_counts contains empty node code")
|
|
normalized[node_code] = _normalize_thread_count(raw_thread_count, field_name=f"node_thread_counts.{node_code}")
|
|
return normalized
|
|
|
|
|
|
def _normalize_node_process_counts(payload: object) -> dict[str, int]:
|
|
if payload in (None, ""):
|
|
return {}
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("node_process_counts must be an object")
|
|
|
|
normalized: dict[str, int] = {}
|
|
for raw_node_code, raw_process_count in payload.items():
|
|
node_code = str(raw_node_code or "").strip()
|
|
if not node_code:
|
|
raise ValueError("node_process_counts contains empty node code")
|
|
normalized[node_code] = _normalize_process_count(
|
|
raw_process_count,
|
|
field_name=f"node_process_counts.{node_code}",
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _load_thread_count_config() -> tuple[int, dict[str, int]]:
|
|
thread_count_payload = read_json("thread_count.json", default={"thread_count": "1000"})
|
|
node_thread_counts_payload = read_json("node_thread_counts.json", default={})
|
|
|
|
try:
|
|
default_thread_count = _normalize_thread_count(thread_count_payload.get("thread_count", 1000))
|
|
except ValueError:
|
|
default_thread_count = 1000
|
|
try:
|
|
node_thread_counts = _normalize_node_thread_counts(node_thread_counts_payload)
|
|
except ValueError:
|
|
node_thread_counts = {}
|
|
|
|
redis_client = get_redis()
|
|
try:
|
|
if redis_thread_count := redis_client.get(REDIS_KEYS["thread_count"]):
|
|
try:
|
|
default_thread_count = _normalize_thread_count(redis_thread_count)
|
|
except ValueError:
|
|
pass
|
|
if redis_node_thread_counts := redis_client.get(REDIS_KEYS["node_thread_counts"]):
|
|
try:
|
|
node_thread_counts = _normalize_node_thread_counts(json.loads(redis_node_thread_counts))
|
|
except ValueError:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
return default_thread_count, node_thread_counts
|
|
|
|
|
|
def _load_process_count_config() -> tuple[int, dict[str, int]]:
|
|
process_count_payload = read_json("process_count.json", default={"process_count": "80"})
|
|
node_process_counts_payload = read_json("node_process_counts.json", default={})
|
|
|
|
try:
|
|
default_process_count = _normalize_process_count(process_count_payload.get("process_count", 80))
|
|
except ValueError:
|
|
default_process_count = 80
|
|
try:
|
|
node_process_counts = _normalize_node_process_counts(node_process_counts_payload)
|
|
except ValueError:
|
|
node_process_counts = {}
|
|
|
|
redis_client = get_redis()
|
|
try:
|
|
if redis_process_count := redis_client.get(REDIS_KEYS["process_count"]):
|
|
try:
|
|
default_process_count = _normalize_process_count(redis_process_count)
|
|
except ValueError:
|
|
pass
|
|
if redis_node_process_counts := redis_client.get(REDIS_KEYS["node_process_counts"]):
|
|
try:
|
|
node_process_counts = _normalize_node_process_counts(json.loads(redis_node_process_counts))
|
|
except ValueError:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
|
|
return default_process_count, node_process_counts
|
|
|
|
|
|
def resolve_thread_count(node_code: str | None = None, settings_payload: dict | None = None) -> dict:
|
|
payload = settings_payload or get_settings_payload()
|
|
default_thread_count = int(payload.get("thread_count", 1000))
|
|
node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", {}))
|
|
normalized_node_code = str(node_code or app_settings.node_code or "").strip()
|
|
|
|
override_thread_count = None
|
|
source = "default"
|
|
effective_thread_count = default_thread_count
|
|
if normalized_node_code and normalized_node_code in node_thread_counts:
|
|
override_thread_count = node_thread_counts[normalized_node_code]
|
|
effective_thread_count = override_thread_count
|
|
source = "node_override"
|
|
|
|
return {
|
|
"node_code": normalized_node_code,
|
|
"default_thread_count": default_thread_count,
|
|
"effective_thread_count": effective_thread_count,
|
|
"override_thread_count": override_thread_count,
|
|
"source": source,
|
|
"node_thread_counts": node_thread_counts,
|
|
}
|
|
|
|
|
|
def resolve_process_count(node_code: str | None = None, settings_payload: dict | None = None) -> dict:
|
|
payload = settings_payload or get_settings_payload()
|
|
default_process_count = int(payload.get("process_count", 80))
|
|
node_process_counts = _normalize_node_process_counts(payload.get("node_process_counts", {}))
|
|
normalized_node_code = str(node_code or app_settings.node_code or "").strip()
|
|
|
|
override_process_count = None
|
|
source = "default"
|
|
effective_process_count = default_process_count
|
|
if normalized_node_code and normalized_node_code in node_process_counts:
|
|
override_process_count = node_process_counts[normalized_node_code]
|
|
effective_process_count = override_process_count
|
|
source = "node_override"
|
|
|
|
return {
|
|
"node_code": normalized_node_code,
|
|
"default_process_count": default_process_count,
|
|
"effective_process_count": effective_process_count,
|
|
"override_process_count": override_process_count,
|
|
"source": source,
|
|
"node_process_counts": node_process_counts,
|
|
}
|
|
|
|
|
|
def get_settings_payload() -> dict:
|
|
detect_options = read_json("detect_options.json", default={})
|
|
proxy_config = read_json("proxy_config.json", default={})
|
|
thread_count, node_thread_counts = _load_thread_count_config()
|
|
process_count, node_process_counts = _load_process_count_config()
|
|
|
|
redis_client = get_redis()
|
|
try:
|
|
if redis_detect_options := redis_client.get(REDIS_KEYS["detect_options"]):
|
|
detect_options = json.loads(redis_detect_options)
|
|
if redis_proxy_config := redis_client.get(REDIS_KEYS["proxy_config"]):
|
|
proxy_config = json.loads(redis_proxy_config)
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"detect_options": detect_options,
|
|
"proxy_config": proxy_config,
|
|
"thread_count": thread_count,
|
|
"node_thread_counts": node_thread_counts,
|
|
"process_count": process_count,
|
|
"node_process_counts": node_process_counts,
|
|
"current_node_code": app_settings.node_code,
|
|
"runtime_settings": get_runtime_settings(),
|
|
}
|
|
|
|
|
|
def get_credentials_payload() -> dict:
|
|
credentials = read_json(
|
|
"credentials.json",
|
|
default={
|
|
"juming": {"email": "", "password": ""},
|
|
"juziseo": {"email": "", "password": ""},
|
|
},
|
|
)
|
|
|
|
redis_client = get_redis()
|
|
try:
|
|
if redis_credentials := redis_client.get(REDIS_KEYS["credentials"]):
|
|
credentials = json.loads(redis_credentials)
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"juming": {
|
|
"email": str(credentials.get("juming", {}).get("email", "")),
|
|
"password": str(credentials.get("juming", {}).get("password", "")),
|
|
},
|
|
"juziseo": {
|
|
"email": str(credentials.get("juziseo", {}).get("email", "")),
|
|
"password": str(credentials.get("juziseo", {}).get("password", "")),
|
|
},
|
|
}
|
|
|
|
|
|
def update_credentials_payload(payload: dict) -> dict:
|
|
current = get_credentials_payload()
|
|
credentials = {
|
|
"juming": {
|
|
"email": str(payload.get("juming", {}).get("email", current["juming"]["email"])),
|
|
"password": str(payload.get("juming", {}).get("password", current["juming"]["password"])),
|
|
},
|
|
"juziseo": {
|
|
"email": str(payload.get("juziseo", {}).get("email", current["juziseo"]["email"])),
|
|
"password": str(payload.get("juziseo", {}).get("password", current["juziseo"]["password"])),
|
|
},
|
|
}
|
|
|
|
write_json("credentials.json", credentials)
|
|
|
|
redis_client = get_redis()
|
|
try:
|
|
redis_client.set(REDIS_KEYS["credentials"], json.dumps(credentials, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:credentials:update", json.dumps(credentials, ensure_ascii=False))
|
|
except Exception:
|
|
pass
|
|
|
|
return credentials
|
|
|
|
|
|
def update_settings_payload(payload: dict) -> dict:
|
|
current = get_settings_payload()
|
|
detect_options = payload.get("detect_options", current["detect_options"])
|
|
proxy_config = payload.get("proxy_config", current["proxy_config"])
|
|
thread_count = _normalize_thread_count(payload.get("thread_count", current["thread_count"]))
|
|
node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", current.get("node_thread_counts", {})))
|
|
process_count = _normalize_process_count(payload.get("process_count", current.get("process_count", 80)))
|
|
node_process_counts = _normalize_node_process_counts(
|
|
payload.get("node_process_counts", current.get("node_process_counts", {}))
|
|
)
|
|
runtime_settings = update_runtime_settings(payload.get("runtime_settings", current["runtime_settings"]))
|
|
|
|
write_json("detect_options.json", detect_options)
|
|
write_json("proxy_config.json", proxy_config)
|
|
write_json("thread_count.json", {"thread_count": str(thread_count)})
|
|
write_json("node_thread_counts.json", node_thread_counts)
|
|
write_json("process_count.json", {"process_count": str(process_count)})
|
|
write_json("node_process_counts.json", node_process_counts)
|
|
redis_client = get_redis()
|
|
try:
|
|
redis_client.set(REDIS_KEYS["detect_options"], json.dumps(detect_options, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:detect_options:update", json.dumps(detect_options, ensure_ascii=False))
|
|
redis_client.set(REDIS_KEYS["proxy_config"], json.dumps(proxy_config, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:proxy_config:update", json.dumps(proxy_config, ensure_ascii=False))
|
|
redis_client.set(REDIS_KEYS["thread_count"], thread_count)
|
|
redis_client.publish("domain_tool:thread_count:update", str(thread_count))
|
|
redis_client.set(REDIS_KEYS["node_thread_counts"], json.dumps(node_thread_counts, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:node_thread_counts:update", json.dumps(node_thread_counts, ensure_ascii=False))
|
|
redis_client.set(REDIS_KEYS["process_count"], process_count)
|
|
redis_client.publish("domain_tool:process_count:update", str(process_count))
|
|
redis_client.set(REDIS_KEYS["node_process_counts"], json.dumps(node_process_counts, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:node_process_counts:update", json.dumps(node_process_counts, ensure_ascii=False))
|
|
redis_client.publish("domain_tool:config_update", "node_thread_counts")
|
|
redis_client.publish("domain_tool:config_update", "node_process_counts")
|
|
redis_client.publish("domain_tool:config_update", "detect_options")
|
|
redis_client.publish("domain_tool:config_update", "proxy_config")
|
|
redis_client.publish("domain_tool:config_update", "thread_count")
|
|
redis_client.publish("domain_tool:config_update", "process_count")
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"detect_options": detect_options,
|
|
"proxy_config": proxy_config,
|
|
"thread_count": thread_count,
|
|
"node_thread_counts": node_thread_counts,
|
|
"process_count": process_count,
|
|
"node_process_counts": node_process_counts,
|
|
"current_node_code": app_settings.node_code,
|
|
"runtime_settings": runtime_settings,
|
|
}
|
|
|
|
|
|
def export_settings_snapshot() -> dict:
|
|
return {
|
|
"schema_version": "1.0",
|
|
"exported_at": datetime.now().isoformat(),
|
|
"source": "domain-api",
|
|
"settings": get_settings_payload(),
|
|
}
|
|
|
|
|
|
def import_settings_snapshot(payload: dict) -> dict:
|
|
settings_payload = payload.get("settings", payload)
|
|
validate_settings_payload(settings_payload)
|
|
backup_current_settings("import")
|
|
return update_settings_payload(settings_payload)
|
|
|
|
|
|
def validate_settings_payload(payload: dict) -> None:
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("invalid settings payload")
|
|
|
|
if "thread_count" in payload:
|
|
_normalize_thread_count(payload["thread_count"])
|
|
|
|
if "node_thread_counts" in payload:
|
|
_normalize_node_thread_counts(payload["node_thread_counts"])
|
|
|
|
if "process_count" in payload:
|
|
_normalize_process_count(payload["process_count"])
|
|
|
|
if "node_process_counts" in payload:
|
|
_normalize_node_process_counts(payload["node_process_counts"])
|
|
|
|
if "detect_options" in payload:
|
|
detect_options = payload["detect_options"]
|
|
if not isinstance(detect_options, dict):
|
|
raise ValueError("detect_options must be an object")
|
|
detect_order = detect_options.get("detect_order", [])
|
|
if detect_order and not isinstance(detect_order, list):
|
|
raise ValueError("detect_order must be an array")
|
|
if isinstance(detect_order, list):
|
|
unknown_keys = [item for item in detect_order if item not in DETECT_OPTION_KEYS]
|
|
if unknown_keys:
|
|
raise ValueError(f"unknown detect option keys: {', '.join(unknown_keys)}")
|
|
|
|
if "proxy_config" in payload:
|
|
proxy_config = payload["proxy_config"]
|
|
if not isinstance(proxy_config, dict):
|
|
raise ValueError("proxy_config must be an object")
|
|
proxy_urls = proxy_config.get("proxy_urls", [])
|
|
if proxy_urls and not isinstance(proxy_urls, list):
|
|
raise ValueError("proxy_urls must be an array")
|
|
|
|
if "runtime_settings" in payload:
|
|
runtime_settings = payload["runtime_settings"]
|
|
if not isinstance(runtime_settings, dict):
|
|
raise ValueError("runtime_settings must be an object")
|
|
worker_mode = runtime_settings.get("worker_mode")
|
|
if worker_mode and worker_mode not in {"windows-local", "linux-systemd"}:
|
|
raise ValueError("worker_mode must be windows-local or linux-systemd")
|
|
for key in ("worker_service_name", "api_service_name", "sync_agent_service_name"):
|
|
if key in runtime_settings and runtime_settings[key] is not None and not str(runtime_settings[key]).strip():
|
|
raise ValueError(f"{key} must not be empty")
|
|
if "worker_log_sync_enabled" in runtime_settings and not isinstance(runtime_settings["worker_log_sync_enabled"], bool):
|
|
raise ValueError("worker_log_sync_enabled must be a boolean")
|
|
if "worker_log_sync_mode" in runtime_settings:
|
|
worker_log_sync_mode = str(runtime_settings.get("worker_log_sync_mode") or "").strip().lower()
|
|
if worker_log_sync_mode not in {"key", "full"}:
|
|
raise ValueError("worker_log_sync_mode must be key or full")
|
|
|
|
|
|
def backup_current_settings(reason: str = "manual") -> dict:
|
|
snapshot = export_settings_snapshot()
|
|
snapshot["backup_reason"] = reason
|
|
filename = f"settings_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
|
path = settings_backup_root() / filename
|
|
path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return {
|
|
"filename": filename,
|
|
"path": str(path),
|
|
}
|
|
|
|
|
|
def list_settings_backups(limit: int = 20) -> list[dict]:
|
|
root = settings_backup_root()
|
|
files = sorted(root.glob("settings_backup_*.json"), key=lambda item: item.stat().st_mtime, reverse=True)
|
|
result: list[dict] = []
|
|
for item in files[:limit]:
|
|
backup_reason = ""
|
|
try:
|
|
payload = json.loads(item.read_text(encoding="utf-8"))
|
|
backup_reason = str(payload.get("backup_reason", ""))
|
|
except Exception:
|
|
backup_reason = ""
|
|
result.append(
|
|
{
|
|
"filename": item.name,
|
|
"path": str(item),
|
|
"size": item.stat().st_size,
|
|
"modified_at": datetime.fromtimestamp(item.stat().st_mtime).isoformat(),
|
|
"backup_reason": backup_reason,
|
|
}
|
|
)
|
|
return result
|