admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
domain-api/app/node_agent.py
View File

@@ -10,6 +10,7 @@ import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
@@ -678,6 +679,265 @@ def _publish_local_config_update(config_type: str) -> None:
return
def _normalize_process_count(value: object, default: int = 1) -> int:
try:
normalized = int(value)
except Exception:
return max(1, int(default or 1))
return max(1, normalized)
def _resolve_desired_process_count(bundle: dict) -> int:
normalized_bundle = dict(bundle or {})
node_process_counts = normalized_bundle.get("node_process_counts")
if isinstance(node_process_counts, dict):
override_value = node_process_counts.get(NODE_CODE)
if override_value not in (None, ""):
return _normalize_process_count(override_value, default=80)
return _normalize_process_count(normalized_bundle.get("process_count", 80), default=80)
def _worker_instance_suffixes(extra_count: int) -> list[str]:
alphabet = "abcdefghijklmnopqrstuvwxyz"
suffixes: list[str] = []
normalized_extra_count = max(0, int(extra_count or 0))
if normalized_extra_count <= 0:
return suffixes
for char in alphabet:
suffixes.append(char)
if len(suffixes) >= normalized_extra_count:
return suffixes
for first in alphabet:
for second in alphabet:
suffixes.append(f"{first}{second}")
if len(suffixes) >= normalized_extra_count:
return suffixes
return suffixes[:normalized_extra_count]
def _worker_env_path(service_name: str, suffix: str = "") -> Path:
normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME
suffix_text = f"-{suffix}" if str(suffix or "").strip() else ""
return Path("/etc/default") / f"{normalized_service_name}{suffix_text}"
def _worker_instance_unit_path(service_name: str) -> Path:
normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME
return Path("/etc/systemd/system") / f"{normalized_service_name}@.service"
def _worker_instance_unit_name(service_name: str, suffix: str) -> str:
normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME
return f"{normalized_service_name}@{suffix}"
def _template_worker_unit_source_path() -> Path:
return Path(_PROJECT_DIR) / "deploy" / "systemd" / "domain-worker@.service"
def _render_worker_instance_env(base_env_text: str, *, instance_node_code: str, parent_node_code: str) -> str:
lines: list[str] = []
saw_node_code = False
saw_parent = False
for raw_line in str(base_env_text or "").splitlines():
if raw_line.startswith("NODE_CODE="):
lines.append(f"NODE_CODE={instance_node_code}")
saw_node_code = True
continue
if raw_line.startswith("WORKER_PARENT_NODE_CODE="):
lines.append(f"WORKER_PARENT_NODE_CODE={parent_node_code}")
saw_parent = True
continue
lines.append(raw_line)
if not saw_node_code:
lines.append(f"NODE_CODE={instance_node_code}")
if not saw_parent:
lines.append(f"WORKER_PARENT_NODE_CODE={parent_node_code}")
return "\n".join(lines).rstrip() + "\n"
def _ensure_worker_instance_unit_template(service_name: str) -> tuple[bool, str]:
unit_path = _worker_instance_unit_path(service_name)
source_path = _template_worker_unit_source_path()
if not source_path.exists():
return False, f"worker instance template missing: {source_path}"
try:
template_text = source_path.read_text(encoding="utf-8")
normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME
rendered = (
template_text
.replace("domaincheck-worker-%i", f"{normalized_service_name}-%i")
.replace("domaincheck-worker@%i", f"{normalized_service_name}@%i")
)
if unit_path.exists():
existing = unit_path.read_text(encoding="utf-8")
if existing == rendered:
return True, str(unit_path)
unit_path.write_text(rendered, encoding="utf-8")
return True, str(unit_path)
except Exception as exc:
return False, f"write worker instance template failed: {exc}"
def _worker_instance_start_batch_size() -> int:
raw_value = str(os.getenv("NODE_AGENT_WORKER_RECONCILE_BATCH_SIZE", "") or "").strip()
try:
return max(1, min(32, int(raw_value or 6)))
except Exception:
return 6
def _worker_instance_start_batch_delay_seconds() -> float:
raw_value = str(os.getenv("NODE_AGENT_WORKER_RECONCILE_BATCH_DELAY_SECONDS", "") or "").strip()
try:
return max(0.0, min(30.0, float(raw_value or 1.0)))
except Exception:
return 1.0
def _chunked_units(units: list[str], size: int) -> list[list[str]]:
batch_size = max(1, int(size or 1))
return [units[index:index + batch_size] for index in range(0, len(units), batch_size)]
def _reconcile_worker_instances(bundle: dict) -> dict:
runtime_settings = dict(bundle.get("runtime_settings") or {})
worker_mode = str(runtime_settings.get("worker_mode") or "").strip() or "windows-local"
worker_service_name = str(runtime_settings.get("worker_service_name") or WORKER_SERVICE_NAME).strip() or WORKER_SERVICE_NAME
desired_process_count = _resolve_desired_process_count(bundle)
if worker_mode != "linux-systemd":
return {
"applied": False,
"reason": f"worker_mode={worker_mode}",
"desired_process_count": desired_process_count,
}
if NODE_REGION != "mainland":
return {
"applied": False,
"reason": f"region={NODE_REGION}",
"desired_process_count": desired_process_count,
}
base_env_path = _worker_env_path(worker_service_name)
if not base_env_path.exists():
return {
"applied": False,
"reason": f"base env missing: {base_env_path}",
"desired_process_count": desired_process_count,
}
ok, template_message = _ensure_worker_instance_unit_template(worker_service_name)
if not ok:
return {
"applied": False,
"reason": template_message,
"desired_process_count": desired_process_count,
}
desired_suffixes = _worker_instance_suffixes(max(0, desired_process_count - 1))
desired_units = [_worker_instance_unit_name(worker_service_name, suffix) for suffix in desired_suffixes]
desired_env_paths = {_worker_env_path(worker_service_name, suffix) for suffix in desired_suffixes}
managed_prefix = f"{worker_service_name}-"
try:
base_env_text = base_env_path.read_text(encoding="utf-8")
for suffix in desired_suffixes:
env_path = _worker_env_path(worker_service_name, suffix)
env_path.write_text(
_render_worker_instance_env(
base_env_text,
instance_node_code=f"{NODE_CODE}-{suffix}",
parent_node_code=NODE_CODE,
),
encoding="utf-8",
)
except Exception as exc:
return {
"applied": False,
"reason": f"write worker env failed: {exc}",
"desired_process_count": desired_process_count,
}
existing_env_paths: list[Path] = []
try:
for candidate in Path("/etc/default").iterdir():
if not candidate.is_file():
continue
if not candidate.name.startswith(managed_prefix):
continue
existing_env_paths.append(candidate)
except Exception:
existing_env_paths = []
stale_env_paths = [
candidate
for candidate in existing_env_paths
if candidate not in desired_env_paths
]
rc, stdout, stderr = _run(["systemctl", "daemon-reload"], timeout=90)
if rc != 0:
return {
"applied": False,
"reason": stderr or stdout or "systemctl daemon-reload failed",
"desired_process_count": desired_process_count,
}
if desired_units:
rc, stdout, stderr = _run(["systemctl", "enable", *desired_units], timeout=180)
if rc != 0:
return {
"applied": False,
"reason": stderr or stdout or "systemctl enable worker instances failed",
"desired_process_count": desired_process_count,
}
batch_size = _worker_instance_start_batch_size()
batch_delay_seconds = _worker_instance_start_batch_delay_seconds()
for batch_index, unit_batch in enumerate(_chunked_units(desired_units, batch_size), start=1):
rc, stdout, stderr = _run(
["systemctl", "start", *unit_batch],
timeout=max(120, 30 * len(unit_batch)),
)
if rc != 0:
return {
"applied": False,
"reason": (
stderr
or stdout
or f"systemctl start worker instances failed at batch {batch_index}"
),
"desired_process_count": desired_process_count,
}
if batch_delay_seconds > 0 and batch_index * batch_size < len(desired_units):
time.sleep(batch_delay_seconds)
stale_units = [
f"{worker_service_name}@{candidate.name[len(managed_prefix):]}"
for candidate in stale_env_paths
if candidate.name[len(managed_prefix):]
]
if stale_units:
_run(["systemctl", "stop", *stale_units], timeout=180)
_run(["systemctl", "disable", *stale_units], timeout=180)
for candidate in stale_env_paths:
try:
candidate.unlink()
except Exception:
continue
return {
"applied": True,
"reason": "reconciled",
"desired_process_count": desired_process_count,
"instance_service_name": f"{worker_service_name}@.service",
"extra_instances": len(desired_suffixes),
"stale_instances_removed": len(stale_units),
"template_path": template_message,
}
def _apply_runtime_config(bundle: dict) -> bool:
global _LAST_RUNTIME_CONFIG_HASH
@@ -688,6 +948,13 @@ def _apply_runtime_config(bundle: dict) -> bool:
json.dumps(normalized_bundle, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
if bundle_hash and bundle_hash == _LAST_RUNTIME_CONFIG_HASH:
reconcile_summary = _reconcile_worker_instances(normalized_bundle)
if reconcile_summary.get("applied"):
_log(
"worker instance reconcile refreshed: "
f"desired={reconcile_summary.get('desired_process_count', 1)} "
f"extra={reconcile_summary.get('extra_instances', 0)}"
)
return False
from app.core.files import write_json
@@ -696,8 +963,10 @@ def _apply_runtime_config(bundle: dict) -> bool:
detect_options = dict(normalized_bundle.get("detect_options") or {})
proxy_config = dict(normalized_bundle.get("proxy_config") or {})
thread_count = int(normalized_bundle.get("thread_count", 2) or 2)
thread_count = int(normalized_bundle.get("thread_count", 1000) or 1000)
node_thread_counts = dict(normalized_bundle.get("node_thread_counts") or {})
process_count = int(normalized_bundle.get("process_count", 80) or 80)
node_process_counts = dict(normalized_bundle.get("node_process_counts") or {})
runtime_settings = dict(normalized_bundle.get("runtime_settings") or {})
sensitive_words = dict(normalized_bundle.get("sensitive_words") or {})
sensitive_words_text = str(sensitive_words.get("text") or "")
@@ -707,6 +976,8 @@ def _apply_runtime_config(bundle: dict) -> bool:
write_json("proxy_config.json", proxy_config)
write_json("thread_count.json", {"thread_count": str(thread_count)})
write_json("node_thread_counts.json", node_thread_counts)
write_json("process_count.json", {"process_count": str(process_count)})
write_json("node_process_counts.json", node_process_counts)
write_json("runtime_settings.json", runtime_settings)
write_json("runtime/runtime_settings.json", runtime_settings)
write_json(
@@ -736,12 +1007,16 @@ def _apply_runtime_config(bundle: dict) -> bool:
redis_client.set("domain_tool:proxy_config", json.dumps(proxy_config, ensure_ascii=False))
redis_client.set("domain_tool:thread_count", thread_count)
redis_client.set("domain_tool:node_thread_counts", json.dumps(node_thread_counts, ensure_ascii=False))
redis_client.set("domain_tool:process_count", process_count)
redis_client.set("domain_tool:node_process_counts", json.dumps(node_process_counts, ensure_ascii=False))
redis_client.set("domain_tool:runtime_settings", json.dumps(runtime_settings, ensure_ascii=False))
redis_client.set("domain_tool:sensitive_words", json.dumps(sensitive_word_items, ensure_ascii=False))
redis_client.publish("domain_tool:config_update", "detect_options")
redis_client.publish("domain_tool:config_update", "proxy_config")
redis_client.publish("domain_tool:config_update", "thread_count")
redis_client.publish("domain_tool:config_update", "node_thread_counts")
redis_client.publish("domain_tool:config_update", "process_count")
redis_client.publish("domain_tool:config_update", "node_process_counts")
redis_client.publish("domain_tool:config_update", "runtime_settings")
redis_client.publish("domain_tool:config_update", "sensitive_words")
except Exception:
@@ -749,16 +1024,30 @@ def _apply_runtime_config(bundle: dict) -> bool:
_publish_local_config_update("proxy_config")
_publish_local_config_update("thread_count")
_publish_local_config_update("node_thread_counts")
_publish_local_config_update("process_count")
_publish_local_config_update("node_process_counts")
_publish_local_config_update("runtime_settings")
_publish_local_config_update("sensitive_words")
reconcile_summary = _reconcile_worker_instances(normalized_bundle)
_LAST_RUNTIME_CONFIG_HASH = bundle_hash
_log(
"runtime config applied: "
f"thread_count={thread_count} "
f"process_count={process_count} "
f"node_override={node_thread_counts.get(NODE_CODE)} "
f"process_override={node_process_counts.get(NODE_CODE)} "
f"sensitive_words={int(sensitive_words.get('total', 0) or 0)}"
)
if reconcile_summary.get("applied"):
_log(
"worker instance reconcile: "
f"desired={reconcile_summary.get('desired_process_count', 1)} "
f"extra={reconcile_summary.get('extra_instances', 0)} "
f"removed={reconcile_summary.get('stale_instances_removed', 0)}"
)
elif reconcile_summary.get("reason"):
_log(f"worker instance reconcile skipped: {reconcile_summary.get('reason')}")
return True
@@ -914,6 +1203,15 @@ def _detect_runtime_snapshot() -> dict:
"phase_detail": phase_detail,
"recent_warning": str(detect_status.get("recent_warning") or "").strip(),
"updated_at": str(runtime_state.get("updated_at") or "").strip(),
"available_proxy_count": int(detect_status.get("available_proxy_count", 0) or 0),
"proxy_runtime_label": str(detect_status.get("proxy_runtime_label") or "").strip(),
"proxy_runtime_reason": str(detect_status.get("proxy_runtime_reason") or "").strip(),
"proxy_last_refresh_status": str(detect_status.get("proxy_last_refresh_status") or "").strip(),
"proxy_last_refresh_time": str(detect_status.get("proxy_last_refresh_time") or "").strip(),
"proxy_last_refresh_source_count": int(detect_status.get("proxy_last_refresh_source_count", 0) or 0),
"proxy_last_refresh_total_items": int(detect_status.get("proxy_last_refresh_total_items", 0) or 0),
"proxy_last_validated_count": int(detect_status.get("proxy_last_validated_count", 0) or 0),
"proxy_last_available_count": int(detect_status.get("proxy_last_available_count", 0) or 0),
"detect_participating": bool(
detect_status.get("detect_participating", False)
or current_load > 0
@@ -934,6 +1232,15 @@ def _detect_runtime_snapshot() -> dict:
"phase_detail": worker_message,
"recent_warning": "",
"updated_at": str(worker_runtime.get("latest_start_time") or "").strip(),
"available_proxy_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_reason": "",
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
"proxy_last_available_count": 0,
"detect_participating": False,
"error": str(exc),
}