feat: add ops center and node onboarding flow
This commit is contained in:
Your Name
345
domain-api/deploy/multi-region/check_env_drift.sh
Normal file
345
domain-api/deploy/multi-region/check_env_drift.sh
Normal file
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
API_SERVICE="${API_SERVICE:-domaincheck-api}"
|
||||
WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
|
||||
SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}"
|
||||
NODE_AGENT_SERVICE="${NODE_AGENT_SERVICE:-domaincheck-node-agent}"
|
||||
API_ENV_FILE="${API_ENV_FILE:-/etc/default/domaincheck-api}"
|
||||
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/default/domaincheck-worker}"
|
||||
NODE_AGENT_ENV_FILE="${NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
API_DIR="${PROJECT_ROOT}/domain-api"
|
||||
WEB_DIR="${PROJECT_ROOT}/domain-web"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
|
||||
FETCH_BODY=""
|
||||
FETCH_STATUS="000"
|
||||
|
||||
fetch_json() {
|
||||
local url="${1:-}"
|
||||
local raw_text
|
||||
raw_text="$(curl -sS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT:-3}" \
|
||||
--max-time "${CURL_MAX_TIME:-10}" \
|
||||
"${url}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
||||
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
|
||||
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
|
||||
}
|
||||
|
||||
echo "[1/8] local command availability"
|
||||
for cmd in bash curl systemctl "${PYTHON_BIN}" node npm; do
|
||||
if command -v "${cmd}" >/dev/null 2>&1; then
|
||||
printf '%s: ok -> %s\n' "${cmd}" "$(command -v "${cmd}")"
|
||||
else
|
||||
printf '%s: missing\n' "${cmd}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[2/8] python environment"
|
||||
"${PYTHON_BIN}" - <<'PY' "${API_DIR}" "${WEB_DIR}"
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
api_dir = pathlib.Path(sys.argv[1])
|
||||
web_dir = pathlib.Path(sys.argv[2])
|
||||
|
||||
def has_module(name: str) -> bool:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
|
||||
payload = {
|
||||
"python_executable": sys.executable,
|
||||
"python_version": sys.version.split()[0],
|
||||
"pytest_installed": has_module("pytest"),
|
||||
"uvicorn_installed": has_module("uvicorn"),
|
||||
"fastapi_installed": has_module("fastapi"),
|
||||
"api_requirements_exists": (api_dir / "requirements.txt").exists(),
|
||||
"web_package_json_exists": (web_dir / "package.json").exists(),
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[3/8] env files"
|
||||
for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${NODE_AGENT_ENV_FILE}"; do
|
||||
if [[ -f "${env_file}" ]]; then
|
||||
echo "exists: ${env_file}"
|
||||
else
|
||||
echo "missing: ${env_file}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[4/8] systemd service states"
|
||||
for service in "${API_SERVICE}" "${WORKER_SERVICE}" "${SYNC_AGENT_SERVICE}" "${NODE_AGENT_SERVICE}"; do
|
||||
if systemctl list-unit-files "${service}.service" --no-pager >/dev/null 2>&1; then
|
||||
printf '%s: enabled=%s active=%s\n' \
|
||||
"${service}" \
|
||||
"$(systemctl is-enabled "${service}" 2>/dev/null || true)" \
|
||||
"$(systemctl is-active "${service}" 2>/dev/null || true)"
|
||||
else
|
||||
printf '%s: not-installed\n' "${service}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[5/8] runtime preflight"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/preflight"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[6/8] runtime readiness"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/readiness"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[7/8] runtime build-info"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[8/8] condensed env audit summary"
|
||||
"${PYTHON_BIN}" - <<'PY' \
|
||||
"${BASE_URL}" \
|
||||
"${API_SERVICE}" \
|
||||
"${WORKER_SERVICE}" \
|
||||
"${SYNC_AGENT_SERVICE}" \
|
||||
"${NODE_AGENT_SERVICE}" \
|
||||
"${API_ENV_FILE}" \
|
||||
"${WORKER_ENV_FILE}" \
|
||||
"${NODE_AGENT_ENV_FILE}" \
|
||||
"${PROJECT_ROOT}"
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from typing import Dict, List
|
||||
|
||||
base_url, api_service, worker_service, sync_agent_service, node_agent_service, api_env, worker_env, node_agent_env, project_root = sys.argv[1:10]
|
||||
project_root_path = pathlib.Path(project_root)
|
||||
|
||||
def has_module(name: str) -> bool:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
|
||||
def inspect_python_runtime(path: str) -> Dict[str, object]:
|
||||
runtime_path = pathlib.Path(path)
|
||||
if not runtime_path.exists():
|
||||
return {}
|
||||
probe = r"""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
|
||||
mods = ["pytest", "fastapi", "uvicorn"]
|
||||
print(json.dumps({
|
||||
"python_executable": sys.executable,
|
||||
"python_version": sys.version.split()[0],
|
||||
"modules": {name: importlib.util.find_spec(name) is not None for name in mods},
|
||||
}, ensure_ascii=False))
|
||||
"""
|
||||
try:
|
||||
raw = subprocess.check_output([str(runtime_path), "-c", probe], text=True, stderr=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except Exception:
|
||||
return {}
|
||||
payload["path"] = str(runtime_path)
|
||||
return payload
|
||||
|
||||
def systemd_state(service: str) -> dict:
|
||||
try:
|
||||
enabled = subprocess.check_output(
|
||||
["systemctl", "is-enabled", service],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
enabled = ""
|
||||
try:
|
||||
active = subprocess.check_output(
|
||||
["systemctl", "is-active", service],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
active = ""
|
||||
return {"enabled": enabled, "active": active}
|
||||
|
||||
def fetch_json(url: str) -> tuple[str, dict]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as resp:
|
||||
status = str(resp.status)
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return "000", {}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except Exception:
|
||||
return status, {}
|
||||
return status, payload.get("data") if isinstance(payload, dict) else {}
|
||||
|
||||
def read_git_value(*args: str) -> str:
|
||||
try:
|
||||
result = subprocess.check_output(
|
||||
["git", "-C", str(project_root_path), *args],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
return result
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
local_commit_sha = read_git_value("rev-parse", "--short=12", "HEAD")
|
||||
local_commit_ref = read_git_value("rev-parse", "--abbrev-ref", "HEAD")
|
||||
ops_agent_service_text = read_text(project_root_path / "domain-api" / "app" / "services" / "ops_agent_service.py")
|
||||
bootstrap_node_agent_text = read_text(project_root_path / "domain-api" / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
|
||||
|
||||
repo_supports_install_command_block = (
|
||||
"install_command_block" in ops_agent_service_text
|
||||
and "_candidate_node_agent_install_paths" in ops_agent_service_text
|
||||
)
|
||||
repo_supports_multi_layout_bootstrap = "resolve_project_root" in bootstrap_node_agent_text
|
||||
|
||||
preflight_status, preflight = fetch_json(f"{base_url}/api/v1/runtime/preflight")
|
||||
readiness_status, readiness = fetch_json(f"{base_url}/api/v1/runtime/readiness")
|
||||
build_status, build = fetch_json(f"{base_url}/api/v1/runtime/build-info")
|
||||
route_surface = build.get("route_surface") if isinstance(build, dict) else {}
|
||||
|
||||
missing = []
|
||||
tooling_items: List[str] = []
|
||||
for env_file in [api_env, worker_env, node_agent_env]:
|
||||
if not pathlib.Path(env_file).exists():
|
||||
missing.append(f"missing_env:{env_file}")
|
||||
|
||||
python_candidates: List[str] = []
|
||||
seen_candidates = set()
|
||||
for candidate in [
|
||||
"/opt/domaincheck/domainCheck/.venv/bin/python",
|
||||
str(pathlib.Path.cwd() / ".venv" / "bin" / "python"),
|
||||
str(pathlib.Path.cwd() / "domain-api" / ".venv" / "bin" / "python"),
|
||||
pathlib.Path(sys.executable).as_posix(),
|
||||
]:
|
||||
if candidate and candidate not in seen_candidates:
|
||||
seen_candidates.add(candidate)
|
||||
python_candidates.append(candidate)
|
||||
|
||||
python_runtimes = [item for item in (inspect_python_runtime(candidate) for candidate in python_candidates) if item]
|
||||
|
||||
def runtime_has_module(module_name: str) -> bool:
|
||||
for runtime in python_runtimes:
|
||||
modules = runtime.get("modules") or {}
|
||||
if bool(modules.get(module_name)):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not runtime_has_module("fastapi"):
|
||||
missing.append("python_module:fastapi")
|
||||
if not runtime_has_module("uvicorn"):
|
||||
missing.append("python_module:uvicorn")
|
||||
if not runtime_has_module("pytest"):
|
||||
tooling_items.append("python_module:pytest")
|
||||
|
||||
runtime_commit_sha = str(build.get("commit_sha") or "").strip() if isinstance(build, dict) else ""
|
||||
runtime_commit_ref = str(build.get("commit_ref") or "").strip() if isinstance(build, dict) else ""
|
||||
runtime_route_missing_keys = [str(item).strip() for item in list((route_surface or {}).get("missing_keys") or []) if str(item).strip()]
|
||||
runtime_expected_paths = {
|
||||
str(key).strip(): str(value).strip()
|
||||
for key, value in dict((route_surface or {}).get("expected_paths") or {}).items()
|
||||
if str(key).strip()
|
||||
}
|
||||
runtime_bootstrap_plan_route_missing = "ops_node_handover_bootstrap_plan" in runtime_route_missing_keys
|
||||
runtime_declares_bootstrap_plan_route = "ops_node_handover_bootstrap_plan" in runtime_expected_paths
|
||||
runtime_repo_commit_drift = bool(local_commit_sha and runtime_commit_sha and local_commit_sha != runtime_commit_sha)
|
||||
runtime_repo_capability_drift = bool(
|
||||
repo_supports_install_command_block
|
||||
and (runtime_bootstrap_plan_route_missing or not runtime_declares_bootstrap_plan_route)
|
||||
)
|
||||
|
||||
recommended_actions: List[str] = []
|
||||
if runtime_repo_capability_drift:
|
||||
recommended_actions.append("systemctl restart domaincheck-api")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01")
|
||||
elif runtime_repo_commit_drift:
|
||||
recommended_actions.append("systemctl restart domaincheck-api")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100")
|
||||
|
||||
payload = {
|
||||
"status": "ready",
|
||||
"headline": "当前环境没有发现新的基础阻断，可继续做上线复核。",
|
||||
"repo": {
|
||||
"project_root": str(project_root_path),
|
||||
"commit_sha": local_commit_sha,
|
||||
"commit_ref": local_commit_ref,
|
||||
"supports_install_command_block": repo_supports_install_command_block,
|
||||
"supports_multi_layout_bootstrap": repo_supports_multi_layout_bootstrap,
|
||||
},
|
||||
"python": {
|
||||
"pytest_installed": runtime_has_module("pytest"),
|
||||
"fastapi_installed": runtime_has_module("fastapi"),
|
||||
"uvicorn_installed": runtime_has_module("uvicorn"),
|
||||
"runtimes": python_runtimes,
|
||||
},
|
||||
"services": {
|
||||
api_service: systemd_state(api_service),
|
||||
worker_service: systemd_state(worker_service),
|
||||
sync_agent_service: systemd_state(sync_agent_service),
|
||||
node_agent_service: systemd_state(node_agent_service),
|
||||
},
|
||||
"runtime": {
|
||||
"preflight_status_code": preflight_status,
|
||||
"preflight_ok": bool((preflight or {}).get("ok", False)),
|
||||
"readiness_status_code": readiness_status,
|
||||
"readiness_status": str((readiness or {}).get("status") or ""),
|
||||
"readiness_summary": str((readiness or {}).get("summary") or ""),
|
||||
"build_status_code": build_status,
|
||||
"build_commit_sha": runtime_commit_sha,
|
||||
"build_commit_ref": runtime_commit_ref,
|
||||
"route_surface_complete": bool((route_surface or {}).get("surface_complete", False)),
|
||||
"route_surface_missing_keys": runtime_route_missing_keys,
|
||||
"route_surface_declares_bootstrap_plan": runtime_declares_bootstrap_plan_route,
|
||||
"repo_commit_drift": runtime_repo_commit_drift,
|
||||
"repo_capability_drift": runtime_repo_capability_drift,
|
||||
"runtime_may_need_restart": runtime_repo_commit_drift or runtime_repo_capability_drift,
|
||||
},
|
||||
"missing_items": missing,
|
||||
"tooling_items": tooling_items,
|
||||
"recommended_actions": recommended_actions,
|
||||
}
|
||||
|
||||
if not bool((preflight or {}).get("ok", False)):
|
||||
payload["status"] = "blocked"
|
||||
payload["headline"] = "runtime/preflight 未通过，先修环境与依赖，再继续上线复核。"
|
||||
elif str((readiness or {}).get("status") or "") == "blocking":
|
||||
payload["status"] = "blocked"
|
||||
payload["headline"] = "runtime/readiness 当前为 blocking，现场仍不适合进入正式发布。"
|
||||
elif runtime_repo_capability_drift:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "本地仓库已经具备新的 Node Agent 接管能力，但运行中的 API 路由面仍旧，当前更像是服务未重启到最新代码。"
|
||||
elif runtime_repo_commit_drift:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "本地仓库 commit 与运行中 API build-info 不一致，建议先重启或重发当前版本，再继续上线复核。"
|
||||
elif not bool((route_surface or {}).get("surface_complete", False)):
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "运行中 API 路由面还不完整，建议先确认当前实例是否已经吃到最新版本。"
|
||||
elif missing:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "基础环境存在缺口，虽然未必阻断运行，但建议先补齐后再交付。"
|
||||
elif tooling_items:
|
||||
payload["status"] = "ready"
|
||||
payload["headline"] = "运行环境可继续上线复核，但本机工具链仍有缺口，建议后续补齐。"
|
||||
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
Reference in New Issue
Block a user