feat: stabilize multi-region runtime sync and worker orchestration

2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
--- a/domain-api/deploy/linux/smoke_test.py
+++ b/domain-api/deploy/linux/smoke_test.py
@@ -8,7 +8,7 @@ from datetime import datetime
 import requests


-def _fetch_json(session: requests.Session, url: str, timeout: int = 15) -> tuple[bool, str, dict | None]:
+def _fetch_json(session: requests.Session, url: str, timeout: int = 25) -> tuple[bool, str, dict | None]:
    try:
        response = session.get(url, timeout=timeout)
        response.raise_for_status()
@@ -45,20 +45,22 @@ def main() -> int:
    checks: list[dict[str, object]] = []

    endpoints = [
-        ("health", f"{base_url}/health"),
-        ("runtime_status", f"{base_url}/api/v1/runtime/status"),
-        ("runtime_preflight", f"{base_url}/api/v1/runtime/preflight"),
-        ("dashboard_overview", f"{base_url}/api/v1/dashboard/overview"),
-        ("settings_export", f"{base_url}/api/v1/settings/export"),
-        ("settings_backups", f"{base_url}/api/v1/settings/backups"),
-        ("detect_status", f"{base_url}/api/v1/detect/status"),
-        ("imports_summary", f"{base_url}/api/v1/imports/summary"),
-        ("exports", f"{base_url}/api/v1/exports"),
-        ("logs_latest", f"{base_url}/api/v1/logs/latest"),
+        ("health", f"{base_url}/health", 25),
+        ("runtime_status", f"{base_url}/api/v1/runtime/status", 25),
+        ("runtime_preflight", f"{base_url}/api/v1/runtime/preflight", 25),
+        # overview merges multiple heavy runtime aggregates and can legitimately
+        # take longer than lightweight health/readiness endpoints on live load.
+        ("dashboard_overview", f"{base_url}/api/v1/dashboard/overview", 25),
+        ("settings_export", f"{base_url}/api/v1/settings/export", 25),
+        ("settings_backups", f"{base_url}/api/v1/settings/backups", 25),
+        ("detect_status", f"{base_url}/api/v1/detect/status", 25),
+        ("imports_summary", f"{base_url}/api/v1/imports/summary", 25),
+        ("exports", f"{base_url}/api/v1/exports", 25),
+        ("logs_latest", f"{base_url}/api/v1/logs/latest", 25),
    ]

-    for name, url in endpoints:
-        ok, message, payload = _fetch_json(session, url)
+    for name, url, timeout in endpoints:
+        ok, message, payload = _fetch_json(session, url, timeout=timeout)
        checks.append(
            {
                "name": name,
--- a/domain-api/deploy/multi-region/README.md
+++ b/domain-api/deploy/multi-region/README.md
@@ -39,6 +39,7 @@
 - `domain-api/deploy/multi-region/drive_ops_action.sh`
 - `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example`
 - `domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example`
+- `domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example`
 - `domain-api/deploy/multi-region/check_cluster.sh`
 - `domain-api/deploy/multi-region/check_node_agent.sh`
 - `domain-api/deploy/multi-region/check_ops_center_stack.sh`
@@ -218,6 +219,32 @@ bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover
 - `runtime/build-info`
 - 一份最终 `status / headline / missing_items / tooling_items` 收口摘要

+## 一点六、单机多进程 Worker 起步方式
+
+如果大陆执行机是高核大内存机器，而单个 `detect_worker.py` 进程还吃不满机器，可以先不要继续堆单进程线程数，优先改成“同机多实例 Worker”：
+
+```bash
+# 1. 复制实例模板，按实例编号准备独立 env
+cp domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-a
+cp domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-b
+
+# 2. 为每个实例设置不同的 NODE_CODE
+#    mainland-controller-01-a
+#    mainland-controller-01-b
+
+# 3. 启用 systemd 模板实例
+cp domain-api/deploy/systemd/domain-worker@.service /etc/systemd/system/domaincheck-worker@.service
+systemctl daemon-reload
+systemctl enable --now domaincheck-worker@a
+systemctl enable --now domaincheck-worker@b
+```
+
+注意：
+
+- 同一台机器上的每个实例必须使用不同的 `NODE_CODE`
+- 每个实例建议先用中等线程数压测，不要直接把单进程线程数拉到极限
+- 这版控制指令已经支持按 `NODE_CODE` 定向，不同实例不会再一起响应同一条本地 Worker 指令
+
 说明：

 - `missing_items` 只保留真正影响部署或运行面的缺口
--- a/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example
@@ -1,5 +1,8 @@
 WORKER_MODE=linux-systemd
 QT_QPA_PLATFORM=offscreen
+
+# 单实例控制节点参考。若同一台大陆大机需要起多个 Worker 进程，
+# 请改用 domaincheck-worker.instance.env.example + domain-worker@.service。
 NODE_CODE=mainland-controller-01
 NODE_REGION=mainland
 NODE_ROLE=control
--- a/domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example
@@ -0,0 +1,40 @@
+WORKER_MODE=linux-systemd
+QT_QPA_PLATFORM=offscreen
+
+# 对应 systemd 模板:
+#   cp domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-a
+#   cp domain-api/deploy/systemd/domain-worker@.service /etc/systemd/system/domaincheck-worker@.service
+#   systemctl daemon-reload
+#   systemctl enable --now domaincheck-worker@a
+#
+# 同一台机器上起多个 worker 实例时，每个实例必须使用不同的 NODE_CODE，
+# 否则运行态、控制指令和集群心跳会互相覆盖。
+NODE_CODE=mainland-controller-01-a
+WORKER_PARENT_NODE_CODE=mainland-controller-01
+NODE_REGION=mainland
+NODE_ROLE=control
+
+# DB
+# DB_HOST=127.0.0.1
+# DB_PORT=5432
+# DB_DATABASE=domain
+# DB_USER=postgres
+# DB_PASSWORD=postgres
+
+# REDIS
+# REDIS_HOST=127.0.0.1
+# REDIS_PORT=6379
+# REDIS_PASSWORD=
+# REDIS_DB=0
+
+# SYNC
+SYNC_PUSH_ENABLED=true
+SYNC_SOURCE_REGION=mainland
+SYNC_TARGET_REGION=overseas
+SYNC_TARGET_API_BASE_URL=http://海外控制面IP:8100/api/v1
+SYNC_SHARED_TOKEN=
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
+
+# 第一版建议每实例先用中等线程数压测，不要直接把单进程线程拉到极限。
+# THREAD_COUNT=1000
--- a/domain-api/deploy/systemd/domain-api.service
+++ b/domain-api/deploy/systemd/domain-api.service
@@ -11,9 +11,10 @@ Environment="WORKER_MODE=linux-systemd"
 Environment="API_HOST=0.0.0.0"
 Environment="API_PORT=8100"
 Environment="DOMAIN_ROOT=/opt/domaincheck/domainCheck"
-ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100
+ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100 --timeout-graceful-shutdown 15
 Restart=always
 RestartSec=5
+TimeoutStopSec=20
 User=www
 Group=www

--- a/domain-api/deploy/systemd/domain-worker.service
+++ b/domain-api/deploy/systemd/domain-worker.service
@@ -4,13 +4,15 @@ After=network.target redis.service postgresql.service

 [Service]
 Type=simple
-WorkingDirectory=/opt/domaincheck/domainCheck
+WorkingDirectory=/opt/domaincheck/current/domainCheck
 EnvironmentFile=-/etc/default/domaincheck-worker
 Environment="WORKER_MODE=linux-systemd"
 Environment="QT_QPA_PLATFORM=offscreen"
-ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/domainCheck/detect_worker.py
+ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py
 Restart=always
 RestartSec=5
+LimitNOFILE=65535
+TasksMax=infinity
 User=www
 Group=www

--- a/domain-api/deploy/systemd/domain-worker@.service
+++ b/domain-api/deploy/systemd/domain-worker@.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=domainCheck Worker Instance %i
+After=network.target redis.service postgresql.service
+
+[Service]
+Type=simple
+WorkingDirectory=/opt/domaincheck/current/domainCheck
+EnvironmentFile=-/etc/default/domaincheck-worker-%i
+Environment="WORKER_MODE=linux-systemd"
+Environment="QT_QPA_PLATFORM=offscreen"
+ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py
+Restart=always
+RestartSec=5
+LimitNOFILE=65535
+TasksMax=infinity
+SyslogIdentifier=domaincheck-worker@%i
+User=www
+Group=www
+
+[Install]
+WantedBy=multi-user.target