feat: add ops center and node onboarding flow

This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions

View File

@@ -0,0 +1,423 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
OVERSEAS_BASE_URL="${2:-}"
VIEW_MODE="${3:-full}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
LOCAL_NODE_AGENT_SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
LOCAL_NODE_AGENT_ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
FETCH_BODY=""
FETCH_STATUS="000"
fetch_json() {
local url="${1:-}"
local raw_text
raw_text="$(curl -sS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"${url}" \
-w $'\nHTTP_STATUS=%{http_code}' || true)"
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
}
run_local_node_agent_probe() {
SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME}" \
ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE}" \
bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check_node_agent.sh"
}
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}"
STACK_JSON="${FETCH_BODY}"
STACK_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/go-live-summary?base_url=${BASE_URL}"
GO_LIVE_API_JSON="${FETCH_BODY}"
GO_LIVE_API_STATUS="${FETCH_STATUS}"
if [[ "${GO_LIVE_API_STATUS}" == 2* ]]; then
if [[ "${VIEW_MODE}" != "summary" ]]; then
echo "[1/1] go-live summary (api)"
fi
printf '%s\n' "${GO_LIVE_API_JSON}"
exit 0
fi
fetch_json "${BASE_URL}/api/v1/ops/contracts"
CONTRACTS_JSON="${FETCH_BODY}"
CONTRACTS_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad"
LAUNCHPAD_JSON="${FETCH_BODY}"
LAUNCHPAD_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
BUILD_INFO_JSON="${FETCH_BODY}"
BUILD_INFO_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/nodes"
NODES_JSON="${FETCH_BODY}"
NODES_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/overview"
OVERVIEW_JSON="${FETCH_BODY}"
OVERVIEW_STATUS="${FETCH_STATUS}"
LOCAL_NODE_AGENT_RAW="$(run_local_node_agent_probe 2>&1 || true)"
if [[ "${VIEW_MODE}" != "summary" ]]; then
echo "[1/7] ops stack diagnosis"
printf 'status=%s\n%s\n\n' "${STACK_STATUS}" "${STACK_JSON}"
echo "[2/7] ops contracts"
printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}"
echo "[3/7] release launchpad"
printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}"
echo "[4/7] runtime build info"
printf 'status=%s\n%s\n\n' "${BUILD_INFO_STATUS}" "${BUILD_INFO_JSON}"
echo "[5/7] managed nodes"
printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}"
echo "[6/7] ops overview"
printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}"
echo "[7/7] local node agent probe"
printf '%s\n\n' "${LOCAL_NODE_AGENT_RAW}"
fi
echo "[8/8] go-live summary"
python3 - <<'PY' \
"${BASE_URL}" \
"${OVERSEAS_BASE_URL}" \
"${STACK_JSON}" \
"${STACK_STATUS}" \
"${CONTRACTS_JSON}" \
"${CONTRACTS_STATUS}" \
"${LAUNCHPAD_JSON}" \
"${LAUNCHPAD_STATUS}" \
"${BUILD_INFO_JSON}" \
"${BUILD_INFO_STATUS}" \
"${NODES_JSON}" \
"${NODES_STATUS}" \
"${OVERVIEW_JSON}" \
"${OVERVIEW_STATUS}" \
"${LOCAL_NODE_AGENT_RAW}" \
"${LOCAL_NODE_AGENT_SERVICE_NAME}"
import json
import sys
def load_payload(raw_text: str) -> dict:
try:
payload = json.loads(str(raw_text or ""))
except Exception:
return {}
return payload if isinstance(payload, dict) else {}
def load_data(raw_text: str) -> dict:
payload = load_payload(raw_text)
data = payload.get("data")
return data if isinstance(data, dict) else {}
def ensure_dict(value):
return value if isinstance(value, dict) else {}
def ensure_list(value):
return value if isinstance(value, list) else []
def ensure_str(value):
return value if isinstance(value, str) else ""
def truthy_http(status_code: str) -> bool:
return str(status_code or "").startswith("2")
base_url = str(sys.argv[1] or "http://127.0.0.1:8100").strip()
overseas_base_url = str(sys.argv[2] or "").strip()
stack_payload = load_data(sys.argv[3])
stack_status_code = str(sys.argv[4] or "000")
contracts_payload = load_data(sys.argv[5])
contracts_status_code = str(sys.argv[6] or "000")
launchpad_payload = load_data(sys.argv[7])
launchpad_status_code = str(sys.argv[8] or "000")
build_payload = load_data(sys.argv[9])
build_status_code = str(sys.argv[10] or "000")
nodes_payload = load_data(sys.argv[11])
nodes_status_code = str(sys.argv[12] or "000")
overview_payload = load_data(sys.argv[13])
overview_status_code = str(sys.argv[14] or "000")
local_node_agent_raw = str(sys.argv[15] or "")
local_node_agent_service_name = str(sys.argv[16] or "domaincheck-node-agent").strip()
diagnosis = ensure_dict(stack_payload.get("diagnosis"))
release_hub = ensure_dict(stack_payload.get("release_hub"))
managed_nodes = ensure_dict(stack_payload.get("managed_nodes"))
runtime_build_info = ensure_dict(stack_payload.get("runtime_build_info")) or build_payload
launchpad_status = ensure_dict(launchpad_payload.get("launchpad_status"))
launchpad_recommended_target_node_code = ensure_str(launchpad_status.get("recommended_target_node_code"))
launchpad_recommended_recovery_label = ensure_str(launchpad_status.get("recommended_recovery_label"))
launchpad_recommended_recovery_summary = ensure_str(launchpad_status.get("recommended_recovery_summary"))
launchpad_onboarding_bootstrap_pending_nodes = int(
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
)
launchpad_onboarding_acceptance_ready_nodes = int(
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
)
contracts = ensure_list(contracts_payload.get("contracts"))
nodes_summary = ensure_dict(nodes_payload.get("summary")) or ensure_dict(managed_nodes.get("summary"))
route_surface = ensure_dict(runtime_build_info.get("route_surface"))
next_step = ensure_dict(diagnosis.get("next_step"))
operator_decision = ensure_dict(diagnosis.get("operator_decision"))
overview_recommendation = ensure_dict(overview_payload.get("recommendation"))
execution_scene = ensure_dict(overview_payload.get("execution_scene"))
log_sync = ensure_dict(execution_scene.get("log_sync"))
local_agent_enabled = ""
local_agent_active = ""
local_agent_installed = False
for line in [item.strip() for item in local_node_agent_raw.splitlines() if item.strip()]:
if line == "not-installed":
local_agent_installed = False
if line == "enabled":
local_agent_enabled = "enabled"
local_agent_installed = True
elif line == "disabled":
local_agent_enabled = "disabled"
local_agent_installed = True
elif line in {"active", "inactive", "failed"}:
local_agent_active = line
local_agent_installed = True
blocking_reasons = []
warnings = []
publish_blocking_reasons = []
publish_warnings = []
stack_status = ensure_str(diagnosis.get("stack_status")) or "unknown"
if stack_status == "blocked":
blocking_reasons.append("stack_diagnosis=blocked")
publish_blocking_reasons.append("stack_diagnosis=blocked")
elif stack_status == "attention":
warnings.append("stack_diagnosis=attention")
publish_warnings.append("stack_diagnosis=attention")
elif not truthy_http(stack_status_code):
blocking_reasons.append("stack_diagnosis_unavailable")
publish_blocking_reasons.append("stack_diagnosis_unavailable")
surface_complete = bool(route_surface.get("surface_complete", False))
if not surface_complete:
missing_keys = [ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)]
blocking_reasons.append(
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
)
publish_blocking_reasons.append(
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
)
if not truthy_http(contracts_status_code) or len(contracts) <= 0:
blocking_reasons.append("contracts_unavailable")
publish_blocking_reasons.append("contracts_unavailable")
launchpad_status_code_value = ensure_str(launchpad_status.get("status")) or ensure_str(release_hub.get("launchpad_status")) or ""
if launchpad_status_code_value == "blocked":
blocking_reasons.append("release_launchpad=blocked")
publish_blocking_reasons.append("release_launchpad=blocked")
elif launchpad_status_code_value == "attention":
warnings.append("release_launchpad=attention")
publish_warnings.append("release_launchpad=attention")
elif not truthy_http(launchpad_status_code):
blocking_reasons.append("release_launchpad_unavailable")
publish_blocking_reasons.append("release_launchpad_unavailable")
if launchpad_onboarding_bootstrap_pending_nodes > 0:
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
if launchpad_onboarding_acceptance_ready_nodes > 0:
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
if managed_enabled > 0 and remote_access_ready == 0:
blocking_reasons.append("remote_agent_ready=0")
publish_blocking_reasons.append("remote_agent_ready=0")
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
if queue_dead_letter_nodes > 0:
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
publish_warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
if local_agent_installed and local_agent_active not in {"", "active"}:
warnings.append(f"{local_node_agent_service_name}={local_agent_active}")
log_sync_enabled = bool(log_sync.get("enabled"))
log_sync_state = ensure_str(log_sync.get("state"))
log_sync_mode = ensure_str(log_sync.get("mode"))
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
missing_participating_node_codes = [
ensure_str(item)
for item in ensure_list(log_sync.get("missing_participating_nodes"))
if ensure_str(item)
]
if log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes == 0:
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
elif log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes < participating_nodes_total:
warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
publish_warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
operator_lane = ensure_str(operator_decision.get("lane"))
operator_title = ensure_str(operator_decision.get("title"))
operator_primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
if not operator_lane:
if blocking_reasons:
operator_lane = "recovery"
elif warnings:
operator_lane = "verification"
else:
operator_lane = "go-live"
if not operator_title:
operator_title = (
ensure_str(overview_recommendation.get("title"))
or ensure_str(launchpad_status.get("status_label"))
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
)
next_step_action_code = (
ensure_str(next_step.get("action_code"))
or ensure_str(overview_recommendation.get("primary_action_code"))
or ensure_str(launchpad_status.get("recommended_action_code"))
)
next_step_reason = (
ensure_str(next_step.get("reason"))
or ensure_str(overview_recommendation.get("reason"))
or "来自 launchpad/overview fallback"
)
if not operator_primary_command_key:
operator_primary_command_key = next_step_action_code
go_live_status = "ready"
if blocking_reasons:
go_live_status = "blocked"
elif warnings:
go_live_status = "attention"
publish_status = "ready"
if publish_blocking_reasons:
publish_status = "blocked"
elif publish_warnings:
publish_status = "attention"
publish_ready = publish_status == "ready"
if publish_status == "ready":
publish_status_label = "可发布"
elif publish_status == "attention":
publish_status_label = "可发布但建议先复核"
else:
publish_status_label = "暂不可发布"
action_commands = {
"handover_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} handover_first_gap",
"view_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} view_first_gap",
"bootstrap_run": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {launchpad_recommended_target_node_code} cli"
if launchpad_recommended_target_node_code
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} bootstrap_run"
),
"run_acceptance": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {launchpad_recommended_target_node_code} cli"
if launchpad_recommended_target_node_code
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_acceptance"
),
"fix_managed_nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} fix_managed_nodes",
"open_worker_logs_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} open_worker_logs_participating",
"run_inspection_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_inspection_participating",
}
recommended_commands = {
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary",
"contracts": f"bash domain-api/deploy/multi-region/check_ops_contracts.sh {base_url}",
"ops_plane": f"bash domain-api/deploy/multi-region/check_ops_plane.sh {base_url}",
"release_hub": f"bash domain-api/deploy/multi-region/check_release_hub.sh {base_url}",
"inspection": f"bash domain-api/deploy/multi-region/check_ops_inspection.sh {base_url}",
"node_agent": "bash domain-api/deploy/multi-region/check_node_agent.sh",
"overview": f"bash domain-api/deploy/multi-region/drive_ops_center.sh overview {base_url}",
"doctor_export": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url} {overseas_base_url}"
if overseas_base_url
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url}"
),
}
if next_step_action_code and next_step_action_code in action_commands:
recommended_commands["next_step"] = action_commands[next_step_action_code]
if log_sync_enabled:
if "open_worker_logs_participating" in action_commands:
recommended_commands["log_sync_logs"] = action_commands["open_worker_logs_participating"]
if "run_inspection_participating" in action_commands:
recommended_commands["log_sync_inspection"] = action_commands["run_inspection_participating"]
summary = {
"base_url": base_url,
"overseas_base_url": overseas_base_url,
"go_live_status": go_live_status,
"publish_ready": publish_ready,
"publish_status": publish_status,
"publish_status_label": publish_status_label,
"stack_status": stack_status,
"contracts_ready": truthy_http(contracts_status_code) and len(contracts) > 0,
"contracts_total": len(contracts),
"launchpad_status": launchpad_status_code_value,
"launchpad_status_label": ensure_str(launchpad_status.get("status_label")),
"launchpad_recommended_action_code": ensure_str(launchpad_status.get("recommended_action_code")),
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
"route_surface_complete": surface_complete,
"route_surface_missing_keys": [
ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)
],
"managed_enabled": managed_enabled,
"remote_access_ready": remote_access_ready,
"queue_dead_letter_nodes": queue_dead_letter_nodes,
"participating_nodes_total": participating_nodes_total,
"log_sync_enabled": log_sync_enabled,
"log_sync_state": log_sync_state,
"log_sync_mode": log_sync_mode,
"log_sync_covered_nodes": covered_participating_nodes,
"log_sync_missing_node_codes": missing_participating_node_codes,
"next_step_action_code": next_step_action_code,
"next_step_reason": next_step_reason,
"operator_lane": operator_lane,
"operator_title": operator_title,
"operator_primary_command_key": operator_primary_command_key,
"local_node_agent": {
"service_name": local_node_agent_service_name,
"installed": local_agent_installed,
"enabled": local_agent_enabled,
"active": local_agent_active,
},
"blocking_reasons": blocking_reasons,
"warnings": warnings,
"publish_blocking_reasons": publish_blocking_reasons,
"publish_warnings": publish_warnings,
"recommended_commands": recommended_commands,
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY