424 lines
18 KiB
Bash
424 lines
18 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
BASE_URL="${1:-http://127.0.0.1:8100}"
|
|
OVERSEAS_BASE_URL="${2:-}"
|
|
VIEW_MODE="${3:-full}"
|
|
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
|
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
|
|
LOCAL_NODE_AGENT_SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
|
|
LOCAL_NODE_AGENT_ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
|
|
|
FETCH_BODY=""
|
|
FETCH_STATUS="000"
|
|
|
|
fetch_json() {
|
|
local url="${1:-}"
|
|
local raw_text
|
|
raw_text="$(curl -sS \
|
|
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
|
--max-time "${CURL_MAX_TIME}" \
|
|
"${url}" \
|
|
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
|
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
|
|
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
|
|
}
|
|
|
|
run_local_node_agent_probe() {
|
|
SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME}" \
|
|
ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE}" \
|
|
bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check_node_agent.sh"
|
|
}
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}"
|
|
STACK_JSON="${FETCH_BODY}"
|
|
STACK_STATUS="${FETCH_STATUS}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/go-live-summary?base_url=${BASE_URL}"
|
|
GO_LIVE_API_JSON="${FETCH_BODY}"
|
|
GO_LIVE_API_STATUS="${FETCH_STATUS}"
|
|
|
|
if [[ "${GO_LIVE_API_STATUS}" == 2* ]]; then
|
|
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
|
echo "[1/1] go-live summary (api)"
|
|
fi
|
|
printf '%s\n' "${GO_LIVE_API_JSON}"
|
|
exit 0
|
|
fi
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/contracts"
|
|
CONTRACTS_JSON="${FETCH_BODY}"
|
|
CONTRACTS_STATUS="${FETCH_STATUS}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad"
|
|
LAUNCHPAD_JSON="${FETCH_BODY}"
|
|
LAUNCHPAD_STATUS="${FETCH_STATUS}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
|
|
BUILD_INFO_JSON="${FETCH_BODY}"
|
|
BUILD_INFO_STATUS="${FETCH_STATUS}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/nodes"
|
|
NODES_JSON="${FETCH_BODY}"
|
|
NODES_STATUS="${FETCH_STATUS}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/overview"
|
|
OVERVIEW_JSON="${FETCH_BODY}"
|
|
OVERVIEW_STATUS="${FETCH_STATUS}"
|
|
|
|
LOCAL_NODE_AGENT_RAW="$(run_local_node_agent_probe 2>&1 || true)"
|
|
|
|
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
|
echo "[1/7] ops stack diagnosis"
|
|
printf 'status=%s\n%s\n\n' "${STACK_STATUS}" "${STACK_JSON}"
|
|
|
|
echo "[2/7] ops contracts"
|
|
printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}"
|
|
|
|
echo "[3/7] release launchpad"
|
|
printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}"
|
|
|
|
echo "[4/7] runtime build info"
|
|
printf 'status=%s\n%s\n\n' "${BUILD_INFO_STATUS}" "${BUILD_INFO_JSON}"
|
|
|
|
echo "[5/7] managed nodes"
|
|
printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}"
|
|
|
|
echo "[6/7] ops overview"
|
|
printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}"
|
|
|
|
echo "[7/7] local node agent probe"
|
|
printf '%s\n\n' "${LOCAL_NODE_AGENT_RAW}"
|
|
fi
|
|
|
|
echo "[8/8] go-live summary"
|
|
python3 - <<'PY' \
|
|
"${BASE_URL}" \
|
|
"${OVERSEAS_BASE_URL}" \
|
|
"${STACK_JSON}" \
|
|
"${STACK_STATUS}" \
|
|
"${CONTRACTS_JSON}" \
|
|
"${CONTRACTS_STATUS}" \
|
|
"${LAUNCHPAD_JSON}" \
|
|
"${LAUNCHPAD_STATUS}" \
|
|
"${BUILD_INFO_JSON}" \
|
|
"${BUILD_INFO_STATUS}" \
|
|
"${NODES_JSON}" \
|
|
"${NODES_STATUS}" \
|
|
"${OVERVIEW_JSON}" \
|
|
"${OVERVIEW_STATUS}" \
|
|
"${LOCAL_NODE_AGENT_RAW}" \
|
|
"${LOCAL_NODE_AGENT_SERVICE_NAME}"
|
|
import json
|
|
import sys
|
|
|
|
|
|
def load_payload(raw_text: str) -> dict:
|
|
try:
|
|
payload = json.loads(str(raw_text or ""))
|
|
except Exception:
|
|
return {}
|
|
return payload if isinstance(payload, dict) else {}
|
|
|
|
|
|
def load_data(raw_text: str) -> dict:
|
|
payload = load_payload(raw_text)
|
|
data = payload.get("data")
|
|
return data if isinstance(data, dict) else {}
|
|
|
|
|
|
def ensure_dict(value):
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def ensure_list(value):
|
|
return value if isinstance(value, list) else []
|
|
|
|
|
|
def ensure_str(value):
|
|
return value if isinstance(value, str) else ""
|
|
|
|
|
|
def truthy_http(status_code: str) -> bool:
|
|
return str(status_code or "").startswith("2")
|
|
|
|
|
|
base_url = str(sys.argv[1] or "http://127.0.0.1:8100").strip()
|
|
overseas_base_url = str(sys.argv[2] or "").strip()
|
|
stack_payload = load_data(sys.argv[3])
|
|
stack_status_code = str(sys.argv[4] or "000")
|
|
contracts_payload = load_data(sys.argv[5])
|
|
contracts_status_code = str(sys.argv[6] or "000")
|
|
launchpad_payload = load_data(sys.argv[7])
|
|
launchpad_status_code = str(sys.argv[8] or "000")
|
|
build_payload = load_data(sys.argv[9])
|
|
build_status_code = str(sys.argv[10] or "000")
|
|
nodes_payload = load_data(sys.argv[11])
|
|
nodes_status_code = str(sys.argv[12] or "000")
|
|
overview_payload = load_data(sys.argv[13])
|
|
overview_status_code = str(sys.argv[14] or "000")
|
|
local_node_agent_raw = str(sys.argv[15] or "")
|
|
local_node_agent_service_name = str(sys.argv[16] or "domaincheck-node-agent").strip()
|
|
|
|
diagnosis = ensure_dict(stack_payload.get("diagnosis"))
|
|
release_hub = ensure_dict(stack_payload.get("release_hub"))
|
|
managed_nodes = ensure_dict(stack_payload.get("managed_nodes"))
|
|
runtime_build_info = ensure_dict(stack_payload.get("runtime_build_info")) or build_payload
|
|
launchpad_status = ensure_dict(launchpad_payload.get("launchpad_status"))
|
|
launchpad_recommended_target_node_code = ensure_str(launchpad_status.get("recommended_target_node_code"))
|
|
launchpad_recommended_recovery_label = ensure_str(launchpad_status.get("recommended_recovery_label"))
|
|
launchpad_recommended_recovery_summary = ensure_str(launchpad_status.get("recommended_recovery_summary"))
|
|
launchpad_onboarding_bootstrap_pending_nodes = int(
|
|
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
|
|
)
|
|
launchpad_onboarding_acceptance_ready_nodes = int(
|
|
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
|
|
)
|
|
contracts = ensure_list(contracts_payload.get("contracts"))
|
|
nodes_summary = ensure_dict(nodes_payload.get("summary")) or ensure_dict(managed_nodes.get("summary"))
|
|
route_surface = ensure_dict(runtime_build_info.get("route_surface"))
|
|
next_step = ensure_dict(diagnosis.get("next_step"))
|
|
operator_decision = ensure_dict(diagnosis.get("operator_decision"))
|
|
overview_recommendation = ensure_dict(overview_payload.get("recommendation"))
|
|
execution_scene = ensure_dict(overview_payload.get("execution_scene"))
|
|
log_sync = ensure_dict(execution_scene.get("log_sync"))
|
|
|
|
local_agent_enabled = ""
|
|
local_agent_active = ""
|
|
local_agent_installed = False
|
|
for line in [item.strip() for item in local_node_agent_raw.splitlines() if item.strip()]:
|
|
if line == "not-installed":
|
|
local_agent_installed = False
|
|
if line == "enabled":
|
|
local_agent_enabled = "enabled"
|
|
local_agent_installed = True
|
|
elif line == "disabled":
|
|
local_agent_enabled = "disabled"
|
|
local_agent_installed = True
|
|
elif line in {"active", "inactive", "failed"}:
|
|
local_agent_active = line
|
|
local_agent_installed = True
|
|
|
|
blocking_reasons = []
|
|
warnings = []
|
|
publish_blocking_reasons = []
|
|
publish_warnings = []
|
|
|
|
stack_status = ensure_str(diagnosis.get("stack_status")) or "unknown"
|
|
if stack_status == "blocked":
|
|
blocking_reasons.append("stack_diagnosis=blocked")
|
|
publish_blocking_reasons.append("stack_diagnosis=blocked")
|
|
elif stack_status == "attention":
|
|
warnings.append("stack_diagnosis=attention")
|
|
publish_warnings.append("stack_diagnosis=attention")
|
|
elif not truthy_http(stack_status_code):
|
|
blocking_reasons.append("stack_diagnosis_unavailable")
|
|
publish_blocking_reasons.append("stack_diagnosis_unavailable")
|
|
|
|
surface_complete = bool(route_surface.get("surface_complete", False))
|
|
if not surface_complete:
|
|
missing_keys = [ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)]
|
|
blocking_reasons.append(
|
|
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
|
|
)
|
|
publish_blocking_reasons.append(
|
|
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
|
|
)
|
|
|
|
if not truthy_http(contracts_status_code) or len(contracts) <= 0:
|
|
blocking_reasons.append("contracts_unavailable")
|
|
publish_blocking_reasons.append("contracts_unavailable")
|
|
|
|
launchpad_status_code_value = ensure_str(launchpad_status.get("status")) or ensure_str(release_hub.get("launchpad_status")) or ""
|
|
if launchpad_status_code_value == "blocked":
|
|
blocking_reasons.append("release_launchpad=blocked")
|
|
publish_blocking_reasons.append("release_launchpad=blocked")
|
|
elif launchpad_status_code_value == "attention":
|
|
warnings.append("release_launchpad=attention")
|
|
publish_warnings.append("release_launchpad=attention")
|
|
elif not truthy_http(launchpad_status_code):
|
|
blocking_reasons.append("release_launchpad_unavailable")
|
|
publish_blocking_reasons.append("release_launchpad_unavailable")
|
|
|
|
if launchpad_onboarding_bootstrap_pending_nodes > 0:
|
|
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
|
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
|
if launchpad_onboarding_acceptance_ready_nodes > 0:
|
|
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
|
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
|
|
|
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
|
|
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
|
|
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
|
|
if managed_enabled > 0 and remote_access_ready == 0:
|
|
blocking_reasons.append("remote_agent_ready=0")
|
|
publish_blocking_reasons.append("remote_agent_ready=0")
|
|
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
|
|
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
|
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
|
|
|
if queue_dead_letter_nodes > 0:
|
|
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
|
publish_warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
|
|
|
if local_agent_installed and local_agent_active not in {"", "active"}:
|
|
warnings.append(f"{local_node_agent_service_name}={local_agent_active}")
|
|
|
|
log_sync_enabled = bool(log_sync.get("enabled"))
|
|
log_sync_state = ensure_str(log_sync.get("state"))
|
|
log_sync_mode = ensure_str(log_sync.get("mode"))
|
|
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
|
|
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
|
missing_participating_node_codes = [
|
|
ensure_str(item)
|
|
for item in ensure_list(log_sync.get("missing_participating_nodes"))
|
|
if ensure_str(item)
|
|
]
|
|
if log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes == 0:
|
|
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
|
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
|
elif log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes < participating_nodes_total:
|
|
warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
|
|
publish_warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
|
|
|
|
operator_lane = ensure_str(operator_decision.get("lane"))
|
|
operator_title = ensure_str(operator_decision.get("title"))
|
|
operator_primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
|
|
if not operator_lane:
|
|
if blocking_reasons:
|
|
operator_lane = "recovery"
|
|
elif warnings:
|
|
operator_lane = "verification"
|
|
else:
|
|
operator_lane = "go-live"
|
|
if not operator_title:
|
|
operator_title = (
|
|
ensure_str(overview_recommendation.get("title"))
|
|
or ensure_str(launchpad_status.get("status_label"))
|
|
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
|
|
)
|
|
|
|
next_step_action_code = (
|
|
ensure_str(next_step.get("action_code"))
|
|
or ensure_str(overview_recommendation.get("primary_action_code"))
|
|
or ensure_str(launchpad_status.get("recommended_action_code"))
|
|
)
|
|
next_step_reason = (
|
|
ensure_str(next_step.get("reason"))
|
|
or ensure_str(overview_recommendation.get("reason"))
|
|
or "来自 launchpad/overview fallback"
|
|
)
|
|
if not operator_primary_command_key:
|
|
operator_primary_command_key = next_step_action_code
|
|
|
|
go_live_status = "ready"
|
|
if blocking_reasons:
|
|
go_live_status = "blocked"
|
|
elif warnings:
|
|
go_live_status = "attention"
|
|
|
|
publish_status = "ready"
|
|
if publish_blocking_reasons:
|
|
publish_status = "blocked"
|
|
elif publish_warnings:
|
|
publish_status = "attention"
|
|
publish_ready = publish_status == "ready"
|
|
if publish_status == "ready":
|
|
publish_status_label = "可发布"
|
|
elif publish_status == "attention":
|
|
publish_status_label = "可发布但建议先复核"
|
|
else:
|
|
publish_status_label = "暂不可发布"
|
|
|
|
action_commands = {
|
|
"handover_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} handover_first_gap",
|
|
"view_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} view_first_gap",
|
|
"bootstrap_run": (
|
|
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {launchpad_recommended_target_node_code} cli"
|
|
if launchpad_recommended_target_node_code
|
|
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} bootstrap_run"
|
|
),
|
|
"run_acceptance": (
|
|
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {launchpad_recommended_target_node_code} cli"
|
|
if launchpad_recommended_target_node_code
|
|
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_acceptance"
|
|
),
|
|
"fix_managed_nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} fix_managed_nodes",
|
|
"open_worker_logs_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} open_worker_logs_participating",
|
|
"run_inspection_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_inspection_participating",
|
|
}
|
|
|
|
recommended_commands = {
|
|
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary",
|
|
"contracts": f"bash domain-api/deploy/multi-region/check_ops_contracts.sh {base_url}",
|
|
"ops_plane": f"bash domain-api/deploy/multi-region/check_ops_plane.sh {base_url}",
|
|
"release_hub": f"bash domain-api/deploy/multi-region/check_release_hub.sh {base_url}",
|
|
"inspection": f"bash domain-api/deploy/multi-region/check_ops_inspection.sh {base_url}",
|
|
"node_agent": "bash domain-api/deploy/multi-region/check_node_agent.sh",
|
|
"overview": f"bash domain-api/deploy/multi-region/drive_ops_center.sh overview {base_url}",
|
|
"doctor_export": (
|
|
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url} {overseas_base_url}"
|
|
if overseas_base_url
|
|
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url}"
|
|
),
|
|
}
|
|
if next_step_action_code and next_step_action_code in action_commands:
|
|
recommended_commands["next_step"] = action_commands[next_step_action_code]
|
|
if log_sync_enabled:
|
|
if "open_worker_logs_participating" in action_commands:
|
|
recommended_commands["log_sync_logs"] = action_commands["open_worker_logs_participating"]
|
|
if "run_inspection_participating" in action_commands:
|
|
recommended_commands["log_sync_inspection"] = action_commands["run_inspection_participating"]
|
|
|
|
summary = {
|
|
"base_url": base_url,
|
|
"overseas_base_url": overseas_base_url,
|
|
"go_live_status": go_live_status,
|
|
"publish_ready": publish_ready,
|
|
"publish_status": publish_status,
|
|
"publish_status_label": publish_status_label,
|
|
"stack_status": stack_status,
|
|
"contracts_ready": truthy_http(contracts_status_code) and len(contracts) > 0,
|
|
"contracts_total": len(contracts),
|
|
"launchpad_status": launchpad_status_code_value,
|
|
"launchpad_status_label": ensure_str(launchpad_status.get("status_label")),
|
|
"launchpad_recommended_action_code": ensure_str(launchpad_status.get("recommended_action_code")),
|
|
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
|
|
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
|
|
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
|
|
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
|
|
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
|
|
"route_surface_complete": surface_complete,
|
|
"route_surface_missing_keys": [
|
|
ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)
|
|
],
|
|
"managed_enabled": managed_enabled,
|
|
"remote_access_ready": remote_access_ready,
|
|
"queue_dead_letter_nodes": queue_dead_letter_nodes,
|
|
"participating_nodes_total": participating_nodes_total,
|
|
"log_sync_enabled": log_sync_enabled,
|
|
"log_sync_state": log_sync_state,
|
|
"log_sync_mode": log_sync_mode,
|
|
"log_sync_covered_nodes": covered_participating_nodes,
|
|
"log_sync_missing_node_codes": missing_participating_node_codes,
|
|
"next_step_action_code": next_step_action_code,
|
|
"next_step_reason": next_step_reason,
|
|
"operator_lane": operator_lane,
|
|
"operator_title": operator_title,
|
|
"operator_primary_command_key": operator_primary_command_key,
|
|
"local_node_agent": {
|
|
"service_name": local_node_agent_service_name,
|
|
"installed": local_agent_installed,
|
|
"enabled": local_agent_enabled,
|
|
"active": local_agent_active,
|
|
},
|
|
"blocking_reasons": blocking_reasons,
|
|
"warnings": warnings,
|
|
"publish_blocking_reasons": publish_blocking_reasons,
|
|
"publish_warnings": publish_warnings,
|
|
"recommended_commands": recommended_commands,
|
|
}
|
|
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
PY
|