#!/usr/bin/env bash set -u -o pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" BASE_URL="${1:-http://127.0.0.1:8100}" DEADLINE_TEXT="${2:-2026-04-20 12:00:00 +0800}" RUNTIME_DIR="${REPO_ROOT}/docs/ops_center_runtime/night_runs" START_TS="$(date +%Y%m%d_%H%M%S)" RUN_ID="night_run_${START_TS}" LOG_FILE="${RUNTIME_DIR}/${RUN_ID}.log" STATE_DIR="${RUNTIME_DIR}/${RUN_ID}" PID_FILE="${RUNTIME_DIR}/${RUN_ID}.pid" REPORT_FILE="${RUNTIME_DIR}/${RUN_ID}_report.md" SUMMARY_JSON="${RUNTIME_DIR}/${RUN_ID}_summary.json" mkdir -p "${RUNTIME_DIR}" "${STATE_DIR}" echo "$$" > "${PID_FILE}" DEADLINE_EPOCH="$(python3 - <<'PY' "${DEADLINE_TEXT}" import datetime import sys text = sys.argv[1] dt = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M:%S %z") print(int(dt.timestamp())) PY )" cycles=0 log_sync_recover_runs=0 inspection_runs=0 inspection_churn_runs=0 quick_rechecks=0 stop_reason="deadline_reached" last_go_live_status="" last_publish_ready="" last_log_sync_state="" last_issue_total="" last_problem_runs_total="" last_launchpad_status="" last_launchpad_action="" last_problem_run_code="" log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S %z')" "$*" | tee -a "${LOG_FILE}" } run_cmd() { log "RUN $*" if "$@" >> "${LOG_FILE}" 2>&1; then log "OK $*" return 0 fi log "ERR $*" return 1 } fetch_json() { local url="$1" local target="$2" if curl -m 20 -s "${url}" > "${target}.tmp"; then mv "${target}.tmp" "${target}" return 0 fi rm -f "${target}.tmp" return 1 } snapshot_state() { local cycle_dir="$1" mkdir -p "${cycle_dir}" fetch_json "${BASE_URL}/api/v1/ops/go-live-summary" "${cycle_dir}/go_live.json" || true fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis" "${cycle_dir}/stack.json" || true fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" "${cycle_dir}/launchpad.json" || true fetch_json "${BASE_URL}/api/v1/ops/playbook-runs" "${cycle_dir}/playbook_runs.json" || true fetch_json "${BASE_URL}/api/v1/ops/nodes/overseas-control-01/scene-log?limit=120&mode=full" "${cycle_dir}/scene_overseas_control_01.json" || true python3 - <<'PY' \ "${cycle_dir}/go_live.json" \ "${cycle_dir}/stack.json" \ "${cycle_dir}/launchpad.json" \ "${cycle_dir}/playbook_runs.json" \ "${cycle_dir}/scene_overseas_control_01.json" \ "${SUMMARY_JSON}" \ "${cycles}" \ > "${cycle_dir}/summary.env" import json import os import sys go_path, stack_path, launch_path, runs_path, scene_path, summary_path, cycles = sys.argv[1:8] def load_json(path): try: with open(path, "r", encoding="utf-8") as fh: return json.load(fh) except Exception: return {} go_data = load_json(go_path).get("data", {}) stack_data = load_json(stack_path).get("data", {}).get("diagnosis", {}) launch_data = load_json(launch_path).get("data", {}) runs_data = load_json(runs_path).get("data", {}) scene_data = load_json(scene_path).get("data", {}) issues = stack_data.get("issues") or [] issue_codes = [str(item.get("code") or "") for item in issues if item.get("code")] problem_runs = runs_data.get("problem_runs") or [] problem_run_code = "" if problem_runs: problem_run_code = str(problem_runs[0].get("run_code") or "") summary = { "cycles": int(cycles), "go_live_status": str(go_data.get("go_live_status") or ""), "publish_ready": bool(go_data.get("publish_ready")), "log_sync_state": str(go_data.get("log_sync_state") or ""), "log_sync_missing_node_codes": go_data.get("log_sync_missing_node_codes") or [], "stack_status": str(stack_data.get("stack_status") or ""), "issue_total": int(stack_data.get("issue_total") or 0), "blocking_issue_total": int(stack_data.get("blocking_issue_total") or 0), "issue_codes": issue_codes, "launchpad_status": str((launch_data.get("launchpad_status") or {}).get("status") or ""), "launchpad_recommended_action": str((launch_data.get("launchpad_status") or {}).get("recommended_action_code") or ""), "problem_runs_total": int(runs_data.get("problem_runs_total") or 0), "problem_run_code": problem_run_code, "scene_status": str(scene_data.get("status") or ""), "scene_line_count": int((scene_data.get("source_summary") or {}).get("line_count") or 0), "generated_at": str(go_data.get("generated_at") or stack_data.get("generated_at") or ""), } with open(summary_path, "w", encoding="utf-8") as fh: json.dump(summary, fh, ensure_ascii=False, indent=2) def emit(key, value): if isinstance(value, bool): value = "true" if value else "false" elif isinstance(value, list): value = ",".join(str(item) for item in value) else: value = str(value) print(f'{key}="{value}"') for key, value in summary.items(): emit(key.upper(), value) PY # shellcheck disable=SC1090 source "${cycle_dir}/summary.env" last_go_live_status="${GO_LIVE_STATUS}" last_publish_ready="${PUBLISH_READY}" last_log_sync_state="${LOG_SYNC_STATE}" last_issue_total="${ISSUE_TOTAL}" last_problem_runs_total="${PROBLEM_RUNS_TOTAL}" last_launchpad_status="${LAUNCHPAD_STATUS}" last_launchpad_action="${LAUNCHPAD_RECOMMENDED_ACTION}" last_problem_run_code="${PROBLEM_RUN_CODE}" } write_report() { python3 - <<'PY' \ "${REPORT_FILE}" \ "${RUN_ID}" \ "${BASE_URL}" \ "${DEADLINE_TEXT}" \ "${stop_reason}" \ "${cycles}" \ "${log_sync_recover_runs}" \ "${inspection_runs}" \ "${inspection_churn_runs}" \ "${quick_rechecks}" \ "${last_go_live_status}" \ "${last_publish_ready}" \ "${last_log_sync_state}" \ "${last_issue_total}" \ "${last_problem_runs_total}" \ "${last_launchpad_status}" \ "${last_launchpad_action}" \ "${last_problem_run_code}" \ "${SUMMARY_JSON}" import json import sys ( report_path, run_id, base_url, deadline_text, stop_reason, cycles, log_sync_recover_runs, inspection_runs, inspection_churn_runs, quick_rechecks, last_go_live_status, last_publish_ready, last_log_sync_state, last_issue_total, last_problem_runs_total, last_launchpad_status, last_launchpad_action, last_problem_run_code, summary_json_path, ) = sys.argv[1:20] summary = {} try: with open(summary_json_path, "r", encoding="utf-8") as fh: summary = json.load(fh) except Exception: summary = {} lines = [ f"# NIGHT RUN REPORT {run_id}", "", f"- Base URL: `{base_url}`", f"- Deadline: `{deadline_text}`", f"- Stop Reason: `{stop_reason}`", f"- Cycles: `{cycles}`", f"- Log Sync Recover Runs: `{log_sync_recover_runs}`", f"- Inspection Runs: `{inspection_runs}`", f"- Inspection Churn Runs: `{inspection_churn_runs}`", f"- Quick Rechecks: `{quick_rechecks}`", "", "## Final Snapshot", "", f"- `go_live_status = {last_go_live_status}`", f"- `publish_ready = {last_publish_ready}`", f"- `log_sync_state = {last_log_sync_state}`", f"- `issue_total = {last_issue_total}`", f"- `problem_runs_total = {last_problem_runs_total}`", f"- `launchpad_status = {last_launchpad_status}`", f"- `launchpad_recommended_action = {last_launchpad_action}`", f"- `problem_run_code = {last_problem_run_code}`", "", "## Summary JSON", "", "```json", json.dumps(summary, ensure_ascii=False, indent=2), "```", ] with open(report_path, "w", encoding="utf-8") as fh: fh.write("\n".join(lines) + "\n") PY } cleanup() { write_report log "night run stopped: reason=${stop_reason}" log "report: ${REPORT_FILE}" rm -f "${PID_FILE}" } trap cleanup EXIT log "night run started: run_id=${RUN_ID}" log "base_url=${BASE_URL}" log "deadline=${DEADLINE_TEXT}" log "state_dir=${STATE_DIR}" while true; do now_epoch="$(date +%s)" if [[ "${now_epoch}" -ge "${DEADLINE_EPOCH}" ]]; then stop_reason="deadline_reached" break fi cycles=$((cycles + 1)) cycle_dir="${STATE_DIR}/cycle_${cycles}" snapshot_state "${cycle_dir}" log "cycle=${cycles} go_live=${last_go_live_status} log_sync=${last_log_sync_state} issue_total=${last_issue_total} problem_runs=${last_problem_runs_total} launchpad=${last_launchpad_status}/${last_launchpad_action} focus_run=${last_problem_run_code}" if [[ "${last_log_sync_state}" != "full_capture" ]]; then log "action: recover log sync coverage" run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" log-sync-recover "${BASE_URL}" full confirm cli/night-pack || true log_sync_recover_runs=$((log_sync_recover_runs + 1)) run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true inspection_runs=$((inspection_runs + 1)) quick_rechecks=$((quick_rechecks + 1)) sleep 45 continue fi if [[ "${last_issue_total}" != "0" && "${last_problem_runs_total}" != "0" && "${inspection_churn_runs}" -lt 3 ]]; then log "action: create fresh safe inspection run to dilute stale problem run window" run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true inspection_runs=$((inspection_runs + 1)) inspection_churn_runs=$((inspection_churn_runs + 1)) sleep 1800 continue fi if [[ "${last_issue_total}" == "0" && "${last_log_sync_state}" == "full_capture" && "${last_problem_runs_total}" == "0" ]]; then stop_reason="signoff_ready_candidate" break fi sleep 3600 done