admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Your Name
2026-04-17 16:17:19 +08:00
parent 0e096947fc
commit bd6bcb240f
17 changed files with 863 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
domain-api/deploy/multi-region/README.md
View File

@@ -25,6 +25,11 @@
- `deploy/multi-region/bootstrap_mainland.sh`
- `deploy/multi-region/check_cluster.sh`
- `deploy/multi-region/check_mainland_controller.sh`
- `deploy/multi-region/check_mainland_worker.sh`
- `deploy/multi-region/check_worker_participation.sh`
- `deploy/multi-region/check_temp_overseas_control.sh`
- `deploy/multi-region/check_temp_link.sh`
- `deploy/multi-region/check_temp_topology.sh`
- `deploy/multi-region/simulate_cluster_node.py`
- `deploy/multi-region/simulate_multi_region.sh`
- `deploy/multi-region/prune_cluster_nodes.py`
@@ -199,13 +204,48 @@ bash deploy/multi-region/check_mainland_controller.sh
这条命令会直接检查：
- `/etc/default/domaincheck-worker` 是否存在
- `/etc/default/domaincheck-api` 是否存在
- `NODE_ROLE=control` 是否正确
- `SYNC_PUSH_ENABLED` / `SYNC_TARGET_API_BASE_URL` 是否已配置
- `domaincheck-api`
- `domaincheck-worker`
- `domaincheck-sync-agent`
是否已启用并处于运行态
如果要判断“在线 Worker 有几台”和“当前真正领任务执行的是哪台”是否一致，可以执行：
```bash
cd /opt/domaincheck/domain-api
bash deploy/multi-region/check_worker_participation.sh
```
这条命令会直接输出：
- `detect_worker_nodes` 当前在线节点概览
- 最新活跃任务的 `claimed_by / status / count`
- 一段压缩摘要 JSON
如果要检查纯大陆 worker 节点自己的配置和服务状态，可以执行：
```bash
cd /opt/domaincheck/domain-api
bash deploy/multi-region/check_mainland_worker.sh
```
如果当前是“大陆 controller + 大陆 worker + 临时海外控制面”的联调拓扑，还可以执行：
```bash
cd /opt/domaincheck/domain-api
bash deploy/multi-region/check_temp_topology.sh http://127.0.0.1:8100 http://152.53.37.118:8100
```
这条命令会同时汇总：
- 大陆控制面的 `runtime/cluster`
- 大陆控制面的 `runtime/readiness`
- 最新活跃任务到底由谁 `claimed_by`
- 临时海外控制面的 `runtime/cluster`
如果当前还无法真正部署到大陆机器，也可以先在国外测试机上做“单机模拟多节点联调”：
```bash

15
domain-api/deploy/multi-region/check_mainland_controller.sh
View File

@@ -1,8 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
ENV_FILE="${1:-/etc/default/domaincheck-worker}"
ENV_FILE="${1:-/etc/default/domaincheck-api}"
WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
API_SERVICE="${API_SERVICE:-domaincheck-api}"
SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
@@ -61,16 +62,22 @@ if errors:
PY
echo
echo "[3/5] systemd status"
echo "[3/6] systemd status"
systemctl is-enabled "$API_SERVICE" || true
systemctl is-active "$API_SERVICE"
systemctl is-enabled "$WORKER_SERVICE" || true
systemctl is-active "$WORKER_SERVICE"
systemctl is-enabled "$SYNC_AGENT_SERVICE" || true
systemctl is-active "$SYNC_AGENT_SERVICE"
echo
echo "[4/5] worker service detail"
echo "[4/6] api service detail"
systemctl status "$API_SERVICE" --no-pager -l | sed -n '1,25p'
echo
echo "[5/6] worker service detail"
systemctl status "$WORKER_SERVICE" --no-pager -l | sed -n '1,25p'
echo
echo "[5/5] sync-agent detail"
echo "[6/6] sync-agent detail"
systemctl status "$SYNC_AGENT_SERVICE" --no-pager -l | sed -n '1,25p'

70
domain-api/deploy/multi-region/check_mainland_worker.sh Executable file
View File

@@ -0,0 +1,70 @@
#!/usr/bin/env bash
set -euo pipefail
ENV_FILE="${1:-/etc/default/domaincheck-worker}"
WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
echo "[1/4] environment file"
if [ ! -f "$ENV_FILE" ]; then
echo "missing env file: $ENV_FILE"
exit 1
fi
echo "env file: $ENV_FILE"
echo
echo "[2/4] key env summary"
"${PYTHON_BIN}" - <<'PY' "$ENV_FILE"
import json
import sys
from pathlib import Path
env_path = Path(sys.argv[1])
values = {}
for line in env_path.read_text(encoding="utf-8").splitlines():
text = line.strip()
if not text or text.startswith("#") or "=" not in text:
continue
key, value = text.split("=", 1)
values[key.strip()] = value.strip()
required = {
"NODE_CODE": values.get("NODE_CODE", ""),
"NODE_REGION": values.get("NODE_REGION", ""),
"NODE_ROLE": values.get("NODE_ROLE", ""),
"DB_HOST": values.get("DB_HOST", ""),
"DB_PORT": values.get("DB_PORT", ""),
"REDIS_HOST": values.get("REDIS_HOST", ""),
"REDIS_PORT": values.get("REDIS_PORT", ""),
"SYNC_TARGET_API_BASE_URL": values.get("SYNC_TARGET_API_BASE_URL", ""),
}
print(json.dumps(required, ensure_ascii=False, indent=2))
errors = []
if required["NODE_REGION"] != "mainland":
errors.append("NODE_REGION 必须为 mainland")
if required["NODE_ROLE"] != "worker":
errors.append("NODE_ROLE 必须为 worker")
if not required["NODE_CODE"]:
errors.append("NODE_CODE 不能为空")
if not required["DB_HOST"]:
errors.append("DB_HOST 不能为空")
if not required["REDIS_HOST"]:
errors.append("REDIS_HOST 不能为空")
if errors:
print()
print("env validation failed:")
for item in errors:
print(f"- {item}")
sys.exit(1)
PY
echo
echo "[3/4] systemd status"
systemctl is-enabled "$WORKER_SERVICE" || true
systemctl is-active "$WORKER_SERVICE"
echo
echo "[4/4] worker service detail"
systemctl status "$WORKER_SERVICE" --no-pager -l | sed -n '1,30p'

80
domain-api/deploy/multi-region/check_temp_link.sh Executable file
View File

@@ -0,0 +1,80 @@
#!/usr/bin/env bash
set -euo pipefail
MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
echo "[1/6] mainland readiness"
MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
echo "${MAINLAND_READINESS}"
echo
echo
echo "[2/6] mainland sync summary"
MAINLAND_SYNC="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/sync-summary")"
echo "${MAINLAND_SYNC}"
echo
echo
echo "[3/6] mainland push sync"
MAINLAND_PUSH="$(curl -fsS -X POST "${MAINLAND_BASE_URL}/api/v1/runtime/actions/push_sync")"
echo "${MAINLAND_PUSH}"
echo
echo
echo "[4/6] overseas cluster"
OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
echo "${OVERSEAS_CLUSTER}"
echo
echo
echo "[5/6] overseas sync summary"
OVERSEAS_SYNC="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/sync-summary")"
echo "${OVERSEAS_SYNC}"
echo
echo
echo "[6/6] condensed summary"
"${PYTHON_BIN}" - <<'PY' "$MAINLAND_READINESS" "$MAINLAND_SYNC" "$MAINLAND_PUSH" "$OVERSEAS_CLUSTER" "$OVERSEAS_SYNC"
import json
import sys
mainland_readiness = json.loads(sys.argv[1]).get("data", {})
mainland_sync = json.loads(sys.argv[2]).get("data", {})
mainland_push = json.loads(sys.argv[3])
overseas_cluster = json.loads(sys.argv[4]).get("data", {})
overseas_sync = json.loads(sys.argv[5]).get("data", {})
cluster_nodes = overseas_cluster.get("nodes") or []
cluster_summary = overseas_cluster.get("summary") or {}
type_counts = overseas_sync.get("type_counts") or {}
status_counts = overseas_sync.get("status_counts") or {}
print(
json.dumps(
{
"mainland": {
"readiness_status": mainland_readiness.get("status", ""),
"readiness_summary": mainland_readiness.get("summary", ""),
"sync_enabled": bool(mainland_sync.get("enabled", False)),
"target_api_base_url": mainland_sync.get("target_api_base_url", ""),
"push_sync_code": mainland_push.get("code"),
"push_sync_message": mainland_push.get("message", ""),
},
"overseas": {
"nodes_total": overseas_cluster.get("nodes_total", 0),
"online_control_nodes": cluster_summary.get("online_control_nodes", 0),
"online_worker_nodes": cluster_summary.get("online_worker_nodes", 0),
"node_codes": [str(item.get("node_code") or "") for item in cluster_nodes],
"runtime_ingest": type_counts.get("runtime_ingest", 0),
"detect_result_ingest": type_counts.get("detect_result_ingest", 0),
"status_received": status_counts.get("received", 0),
"status_success": status_counts.get("success", 0),
},
},
ensure_ascii=False,
indent=2,
)
)
PY

75
domain-api/deploy/multi-region/check_temp_overseas_control.sh Executable file
View File

@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
echo "[1/5] health"
curl -fsS "${BASE_URL}/health"
echo
echo
echo "[2/5] runtime status"
STATUS_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/status")"
echo "${STATUS_JSON}"
echo
echo
echo "[3/5] runtime cluster"
curl -fsS "${BASE_URL}/api/v1/runtime/cluster"
echo
echo
echo "[4/5] runtime readiness"
curl -fsS "${BASE_URL}/api/v1/runtime/readiness"
echo
echo
echo "[5/5] condensed summary"
"${PYTHON_BIN}" - <<'PY' "$STATUS_JSON"
import json
import sys
data = json.loads(sys.argv[1]).get("data", {})
node = data.get("node") or {}
worker = data.get("worker") or {}
sync_agent = data.get("sync_agent") or {}
detect = data.get("detect") or {}
cluster = data.get("cluster") or {}
summary = cluster.get("summary") or {}
print(
json.dumps(
{
"node": {
"code": node.get("code", ""),
"region": node.get("region", ""),
"role": node.get("role", ""),
},
"worker": {
"running": bool(worker.get("running", False)),
"expected_on_this_node": bool(worker.get("expected_on_this_node", True)),
"message": worker.get("message", ""),
},
"sync_agent": {
"running": bool(sync_agent.get("running", False)),
"expected_on_this_node": bool(sync_agent.get("expected_on_this_node", False)),
"message": sync_agent.get("message", ""),
},
"detect": {
"phase_label": detect.get("phase_label", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"active_job": (detect.get("active_job") or {}).get("job_code", ""),
},
"cluster": {
"nodes_total": cluster.get("nodes_total", 0),
"online_control_nodes": summary.get("online_control_nodes", 0),
"online_worker_nodes": summary.get("online_worker_nodes", 0),
"dedicated_online_worker_nodes": summary.get("dedicated_online_worker_nodes", 0),
},
},
ensure_ascii=False,
indent=2,
)
)
PY

126
domain-api/deploy/multi-region/check_temp_topology.sh Executable file
View File

@@ -0,0 +1,126 @@
#!/usr/bin/env bash
set -euo pipefail
MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
echo "[1/5] mainland runtime cluster"
MAINLAND_CLUSTER="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
echo "${MAINLAND_CLUSTER}"
echo
echo
echo "[2/5] mainland runtime readiness"
MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
echo "${MAINLAND_READINESS}"
echo
echo
echo "[3/5] mainland worker participation"
PARTICIPATION_JSON="$(/www/server/pgsql/bin/psql -U postgres -d domain -At <<'SQL'
with latest_job as (
select id, job_code, status
from detect_jobs
where status in ('pending','running')
order by id desc
limit 1
),
node_stats as (
select
node_code,
role,
status,
coalesce(metadata_json->>'worker_online', '') as worker_online,
coalesce(metadata_json->>'detect_participating', '') as detect_participating
from detect_worker_nodes
),
item_stats as (
select
coalesce(claimed_by, '') as claimed_by,
status,
count(*) as cnt
from detect_job_items
where job_id = (select id from latest_job)
group by claimed_by, status
)
select json_build_object(
'job', (
select json_build_object(
'job_code', coalesce(job_code, ''),
'status', coalesce(status, '')
)
from latest_job
),
'online_nodes', (
select coalesce(json_agg(json_build_object(
'node_code', node_code,
'role', role,
'status', status,
'worker_online', worker_online,
'detect_participating', detect_participating
) order by node_code), '[]'::json)
from node_stats
where status in ('online', 'busy')
),
'claimed_distribution', (
select coalesce(json_agg(json_build_object(
'claimed_by', claimed_by,
'status', status,
'count', cnt
) order by claimed_by, status), '[]'::json)
from item_stats
)
);
SQL
)"
echo "${PARTICIPATION_JSON}"
echo
echo
echo "[4/5] overseas runtime cluster"
OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
echo "${OVERSEAS_CLUSTER}"
echo
echo
echo "[5/5] condensed summary"
"${PYTHON_BIN}" - <<'PY' "$MAINLAND_CLUSTER" "$MAINLAND_READINESS" "$PARTICIPATION_JSON" "$OVERSEAS_CLUSTER"
import json
import sys
mainland_cluster = json.loads(sys.argv[1]).get("data", {})
mainland_readiness = json.loads(sys.argv[2]).get("data", {})
participation = json.loads(sys.argv[3])
overseas_cluster = json.loads(sys.argv[4]).get("data", {})
mainland_summary = mainland_cluster.get("summary") or {}
overseas_summary = overseas_cluster.get("summary") or {}
def node_codes(cluster):
return [str(item.get("node_code") or "") for item in (cluster.get("nodes") or [])]
print(
json.dumps(
{
"mainland": {
"readiness_status": mainland_readiness.get("status", ""),
"readiness_summary": mainland_readiness.get("summary", ""),
"online_worker_nodes": mainland_summary.get("online_worker_nodes", 0),
"dedicated_online_worker_nodes": mainland_summary.get("dedicated_online_worker_nodes", 0),
"online_control_nodes": mainland_summary.get("online_control_nodes", 0),
"node_codes": node_codes(mainland_cluster),
},
"participation": participation,
"overseas": {
"online_worker_nodes": overseas_summary.get("online_worker_nodes", 0),
"dedicated_online_worker_nodes": overseas_summary.get("dedicated_online_worker_nodes", 0),
"online_control_nodes": overseas_summary.get("online_control_nodes", 0),
"node_codes": node_codes(overseas_cluster),
},
},
ensure_ascii=False,
indent=2,
)
)
PY

97
domain-api/deploy/multi-region/check_worker_participation.sh Executable file
View File

@@ -0,0 +1,97 @@
#!/usr/bin/env bash
set -euo pipefail
DB_NAME="${DB_NAME:-domain}"
DB_USER="${DB_USER:-postgres}"
PSQL_BIN="${PSQL_BIN:-/www/server/pgsql/bin/psql}"
echo "[1/3] cluster nodes"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
select
node_code || '|' ||
role || '|' ||
status || '|' ||
coalesce(current_load, 0) || '|' ||
to_char(last_heartbeat_at, 'YYYY-MM-DD HH24:MI:SS') || '|' ||
coalesce(metadata_json->>'worker_online', '') || '|' ||
coalesce(metadata_json->>'detect_participating', '')
from detect_worker_nodes
order by node_code;
"
echo
echo "[2/3] active job distribution"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
select
coalesce(claimed_by, '') || '|' ||
status || '|' ||
count(*)
from detect_job_items
where job_id = (
select id
from detect_jobs
where status in ('pending','running')
order by id desc
limit 1
)
group by claimed_by, status
order by claimed_by, status;
"
echo
echo "[3/3] condensed summary"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -At <<'SQL'
with latest_job as (
select id, job_code, status
from detect_jobs
where status in ('pending','running')
order by id desc
limit 1
),
node_stats as (
select
node_code,
role,
status,
coalesce(metadata_json->>'worker_online', '') as worker_online,
coalesce(metadata_json->>'detect_participating', '') as detect_participating
from detect_worker_nodes
),
item_stats as (
select
coalesce(claimed_by, '') as claimed_by,
status,
count(*) as cnt
from detect_job_items
where job_id = (select id from latest_job)
group by claimed_by, status
)
select json_build_object(
'job', (
select json_build_object(
'job_code', coalesce(job_code, ''),
'status', coalesce(status, '')
)
from latest_job
),
'online_nodes', (
select coalesce(json_agg(json_build_object(
'node_code', node_code,
'role', role,
'status', status,
'worker_online', worker_online,
'detect_participating', detect_participating
) order by node_code), '[]'::json)
from node_stats
where status in ('online', 'busy')
),
'claimed_distribution', (
select coalesce(json_agg(json_build_object(
'claimed_by', claimed_by,
'status', status,
'count', cnt
) order by claimed_by, status), '[]'::json)
from item_stats
)
);
SQL