304 lines
9.3 KiB
Bash
Executable File
304 lines
9.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -u -o pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
|
BASE_URL="${1:-http://127.0.0.1:8100}"
|
|
DEADLINE_TEXT="${2:-2026-04-20 12:00:00 +0800}"
|
|
RUNTIME_DIR="${REPO_ROOT}/docs/ops_center_runtime/night_runs"
|
|
START_TS="$(date +%Y%m%d_%H%M%S)"
|
|
RUN_ID="night_run_${START_TS}"
|
|
LOG_FILE="${RUNTIME_DIR}/${RUN_ID}.log"
|
|
STATE_DIR="${RUNTIME_DIR}/${RUN_ID}"
|
|
PID_FILE="${RUNTIME_DIR}/${RUN_ID}.pid"
|
|
REPORT_FILE="${RUNTIME_DIR}/${RUN_ID}_report.md"
|
|
SUMMARY_JSON="${RUNTIME_DIR}/${RUN_ID}_summary.json"
|
|
|
|
mkdir -p "${RUNTIME_DIR}" "${STATE_DIR}"
|
|
echo "$$" > "${PID_FILE}"
|
|
|
|
DEADLINE_EPOCH="$(python3 - <<'PY' "${DEADLINE_TEXT}"
|
|
import datetime
|
|
import sys
|
|
|
|
text = sys.argv[1]
|
|
dt = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M:%S %z")
|
|
print(int(dt.timestamp()))
|
|
PY
|
|
)"
|
|
|
|
cycles=0
|
|
log_sync_recover_runs=0
|
|
inspection_runs=0
|
|
inspection_churn_runs=0
|
|
quick_rechecks=0
|
|
stop_reason="deadline_reached"
|
|
last_go_live_status=""
|
|
last_publish_ready=""
|
|
last_log_sync_state=""
|
|
last_issue_total=""
|
|
last_problem_runs_total=""
|
|
last_launchpad_status=""
|
|
last_launchpad_action=""
|
|
last_problem_run_code=""
|
|
|
|
log() {
|
|
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S %z')" "$*" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
run_cmd() {
|
|
log "RUN $*"
|
|
if "$@" >> "${LOG_FILE}" 2>&1; then
|
|
log "OK $*"
|
|
return 0
|
|
fi
|
|
log "ERR $*"
|
|
return 1
|
|
}
|
|
|
|
fetch_json() {
|
|
local url="$1"
|
|
local target="$2"
|
|
if curl -m 20 -s "${url}" > "${target}.tmp"; then
|
|
mv "${target}.tmp" "${target}"
|
|
return 0
|
|
fi
|
|
rm -f "${target}.tmp"
|
|
return 1
|
|
}
|
|
|
|
snapshot_state() {
|
|
local cycle_dir="$1"
|
|
mkdir -p "${cycle_dir}"
|
|
|
|
fetch_json "${BASE_URL}/api/v1/ops/go-live-summary" "${cycle_dir}/go_live.json" || true
|
|
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis" "${cycle_dir}/stack.json" || true
|
|
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" "${cycle_dir}/launchpad.json" || true
|
|
fetch_json "${BASE_URL}/api/v1/ops/playbook-runs" "${cycle_dir}/playbook_runs.json" || true
|
|
fetch_json "${BASE_URL}/api/v1/ops/nodes/overseas-control-01/scene-log?limit=120&mode=full" "${cycle_dir}/scene_overseas_control_01.json" || true
|
|
|
|
python3 - <<'PY' \
|
|
"${cycle_dir}/go_live.json" \
|
|
"${cycle_dir}/stack.json" \
|
|
"${cycle_dir}/launchpad.json" \
|
|
"${cycle_dir}/playbook_runs.json" \
|
|
"${cycle_dir}/scene_overseas_control_01.json" \
|
|
"${SUMMARY_JSON}" \
|
|
"${cycles}" \
|
|
> "${cycle_dir}/summary.env"
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
go_path, stack_path, launch_path, runs_path, scene_path, summary_path, cycles = sys.argv[1:8]
|
|
|
|
def load_json(path):
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
except Exception:
|
|
return {}
|
|
|
|
go_data = load_json(go_path).get("data", {})
|
|
stack_data = load_json(stack_path).get("data", {}).get("diagnosis", {})
|
|
launch_data = load_json(launch_path).get("data", {})
|
|
runs_data = load_json(runs_path).get("data", {})
|
|
scene_data = load_json(scene_path).get("data", {})
|
|
|
|
issues = stack_data.get("issues") or []
|
|
issue_codes = [str(item.get("code") or "") for item in issues if item.get("code")]
|
|
problem_runs = runs_data.get("problem_runs") or []
|
|
problem_run_code = ""
|
|
if problem_runs:
|
|
problem_run_code = str(problem_runs[0].get("run_code") or "")
|
|
|
|
summary = {
|
|
"cycles": int(cycles),
|
|
"go_live_status": str(go_data.get("go_live_status") or ""),
|
|
"publish_ready": bool(go_data.get("publish_ready")),
|
|
"log_sync_state": str(go_data.get("log_sync_state") or ""),
|
|
"log_sync_missing_node_codes": go_data.get("log_sync_missing_node_codes") or [],
|
|
"stack_status": str(stack_data.get("stack_status") or ""),
|
|
"issue_total": int(stack_data.get("issue_total") or 0),
|
|
"blocking_issue_total": int(stack_data.get("blocking_issue_total") or 0),
|
|
"issue_codes": issue_codes,
|
|
"launchpad_status": str((launch_data.get("launchpad_status") or {}).get("status") or ""),
|
|
"launchpad_recommended_action": str((launch_data.get("launchpad_status") or {}).get("recommended_action_code") or ""),
|
|
"problem_runs_total": int(runs_data.get("problem_runs_total") or 0),
|
|
"problem_run_code": problem_run_code,
|
|
"scene_status": str(scene_data.get("status") or ""),
|
|
"scene_line_count": int((scene_data.get("source_summary") or {}).get("line_count") or 0),
|
|
"generated_at": str(go_data.get("generated_at") or stack_data.get("generated_at") or ""),
|
|
}
|
|
|
|
with open(summary_path, "w", encoding="utf-8") as fh:
|
|
json.dump(summary, fh, ensure_ascii=False, indent=2)
|
|
|
|
def emit(key, value):
|
|
if isinstance(value, bool):
|
|
value = "true" if value else "false"
|
|
elif isinstance(value, list):
|
|
value = ",".join(str(item) for item in value)
|
|
else:
|
|
value = str(value)
|
|
print(f'{key}="{value}"')
|
|
|
|
for key, value in summary.items():
|
|
emit(key.upper(), value)
|
|
PY
|
|
|
|
# shellcheck disable=SC1090
|
|
source "${cycle_dir}/summary.env"
|
|
last_go_live_status="${GO_LIVE_STATUS}"
|
|
last_publish_ready="${PUBLISH_READY}"
|
|
last_log_sync_state="${LOG_SYNC_STATE}"
|
|
last_issue_total="${ISSUE_TOTAL}"
|
|
last_problem_runs_total="${PROBLEM_RUNS_TOTAL}"
|
|
last_launchpad_status="${LAUNCHPAD_STATUS}"
|
|
last_launchpad_action="${LAUNCHPAD_RECOMMENDED_ACTION}"
|
|
last_problem_run_code="${PROBLEM_RUN_CODE}"
|
|
}
|
|
|
|
write_report() {
|
|
python3 - <<'PY' \
|
|
"${REPORT_FILE}" \
|
|
"${RUN_ID}" \
|
|
"${BASE_URL}" \
|
|
"${DEADLINE_TEXT}" \
|
|
"${stop_reason}" \
|
|
"${cycles}" \
|
|
"${log_sync_recover_runs}" \
|
|
"${inspection_runs}" \
|
|
"${inspection_churn_runs}" \
|
|
"${quick_rechecks}" \
|
|
"${last_go_live_status}" \
|
|
"${last_publish_ready}" \
|
|
"${last_log_sync_state}" \
|
|
"${last_issue_total}" \
|
|
"${last_problem_runs_total}" \
|
|
"${last_launchpad_status}" \
|
|
"${last_launchpad_action}" \
|
|
"${last_problem_run_code}" \
|
|
"${SUMMARY_JSON}"
|
|
import json
|
|
import sys
|
|
|
|
(
|
|
report_path,
|
|
run_id,
|
|
base_url,
|
|
deadline_text,
|
|
stop_reason,
|
|
cycles,
|
|
log_sync_recover_runs,
|
|
inspection_runs,
|
|
inspection_churn_runs,
|
|
quick_rechecks,
|
|
last_go_live_status,
|
|
last_publish_ready,
|
|
last_log_sync_state,
|
|
last_issue_total,
|
|
last_problem_runs_total,
|
|
last_launchpad_status,
|
|
last_launchpad_action,
|
|
last_problem_run_code,
|
|
summary_json_path,
|
|
) = sys.argv[1:20]
|
|
|
|
summary = {}
|
|
try:
|
|
with open(summary_json_path, "r", encoding="utf-8") as fh:
|
|
summary = json.load(fh)
|
|
except Exception:
|
|
summary = {}
|
|
|
|
lines = [
|
|
f"# NIGHT RUN REPORT {run_id}",
|
|
"",
|
|
f"- Base URL: `{base_url}`",
|
|
f"- Deadline: `{deadline_text}`",
|
|
f"- Stop Reason: `{stop_reason}`",
|
|
f"- Cycles: `{cycles}`",
|
|
f"- Log Sync Recover Runs: `{log_sync_recover_runs}`",
|
|
f"- Inspection Runs: `{inspection_runs}`",
|
|
f"- Inspection Churn Runs: `{inspection_churn_runs}`",
|
|
f"- Quick Rechecks: `{quick_rechecks}`",
|
|
"",
|
|
"## Final Snapshot",
|
|
"",
|
|
f"- `go_live_status = {last_go_live_status}`",
|
|
f"- `publish_ready = {last_publish_ready}`",
|
|
f"- `log_sync_state = {last_log_sync_state}`",
|
|
f"- `issue_total = {last_issue_total}`",
|
|
f"- `problem_runs_total = {last_problem_runs_total}`",
|
|
f"- `launchpad_status = {last_launchpad_status}`",
|
|
f"- `launchpad_recommended_action = {last_launchpad_action}`",
|
|
f"- `problem_run_code = {last_problem_run_code}`",
|
|
"",
|
|
"## Summary JSON",
|
|
"",
|
|
"```json",
|
|
json.dumps(summary, ensure_ascii=False, indent=2),
|
|
"```",
|
|
]
|
|
|
|
with open(report_path, "w", encoding="utf-8") as fh:
|
|
fh.write("\n".join(lines) + "\n")
|
|
PY
|
|
}
|
|
|
|
cleanup() {
|
|
write_report
|
|
log "night run stopped: reason=${stop_reason}"
|
|
log "report: ${REPORT_FILE}"
|
|
rm -f "${PID_FILE}"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
|
|
log "night run started: run_id=${RUN_ID}"
|
|
log "base_url=${BASE_URL}"
|
|
log "deadline=${DEADLINE_TEXT}"
|
|
log "state_dir=${STATE_DIR}"
|
|
|
|
while true; do
|
|
now_epoch="$(date +%s)"
|
|
if [[ "${now_epoch}" -ge "${DEADLINE_EPOCH}" ]]; then
|
|
stop_reason="deadline_reached"
|
|
break
|
|
fi
|
|
|
|
cycles=$((cycles + 1))
|
|
cycle_dir="${STATE_DIR}/cycle_${cycles}"
|
|
snapshot_state "${cycle_dir}"
|
|
|
|
log "cycle=${cycles} go_live=${last_go_live_status} log_sync=${last_log_sync_state} issue_total=${last_issue_total} problem_runs=${last_problem_runs_total} launchpad=${last_launchpad_status}/${last_launchpad_action} focus_run=${last_problem_run_code}"
|
|
|
|
if [[ "${last_log_sync_state}" != "full_capture" ]]; then
|
|
log "action: recover log sync coverage"
|
|
run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" log-sync-recover "${BASE_URL}" full confirm cli/night-pack || true
|
|
log_sync_recover_runs=$((log_sync_recover_runs + 1))
|
|
run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true
|
|
inspection_runs=$((inspection_runs + 1))
|
|
quick_rechecks=$((quick_rechecks + 1))
|
|
sleep 45
|
|
continue
|
|
fi
|
|
|
|
if [[ "${last_issue_total}" != "0" && "${last_problem_runs_total}" != "0" && "${inspection_churn_runs}" -lt 3 ]]; then
|
|
log "action: create fresh safe inspection run to dilute stale problem run window"
|
|
run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true
|
|
inspection_runs=$((inspection_runs + 1))
|
|
inspection_churn_runs=$((inspection_churn_runs + 1))
|
|
sleep 1800
|
|
continue
|
|
fi
|
|
|
|
if [[ "${last_issue_total}" == "0" && "${last_log_sync_state}" == "full_capture" && "${last_problem_runs_total}" == "0" ]]; then
|
|
stop_reason="signoff_ready_candidate"
|
|
break
|
|
fi
|
|
|
|
sleep 3600
|
|
done
|