d

2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
--- a/domain-api/deploy/multi-region/fix_mainland_release_base.sh
+++ b/domain-api/deploy/multi-region/fix_mainland_release_base.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROLE="${1:-}"
+BASE_DIR="${2:-/opt/domaincheck}"
+WWW_USER="${WWW_USER:-www}"
+WWW_GROUP="${WWW_GROUP:-www}"
+
+usage() {
+  cat <<'EOF'
+usage:
+  bash domain-api/deploy/multi-region/fix_mainland_release_base.sh <worker|control> [base_dir]
+
+examples:
+  bash domain-api/deploy/multi-region/fix_mainland_release_base.sh worker
+  bash domain-api/deploy/multi-region/fix_mainland_release_base.sh control
+EOF
+}
+
+if [[ "${ROLE}" != "worker" && "${ROLE}" != "control" ]]; then
+  usage >&2
+  exit 1
+fi
+
+if [[ "$(id -u)" != "0" ]]; then
+  echo "please run as root" >&2
+  exit 1
+fi
+
+require_path() {
+  local path="$1"
+  if [[ ! -e "${path}" ]]; then
+    echo "required path missing: ${path}" >&2
+    exit 1
+  fi
+}
+
+install_dropin() {
+  local service_name="$1"
+  local content="$2"
+  local dropin_dir="/etc/systemd/system/${service_name}.d"
+  local dropin_file="${dropin_dir}/current-path.conf"
+  mkdir -p "${dropin_dir}"
+  printf '%s\n' "${content}" > "${dropin_file}"
+  echo "installed ${dropin_file}"
+}
+
+echo "[1/6] validate existing runtime"
+require_path "${BASE_DIR}"
+require_path "${BASE_DIR}/domainCheck"
+require_path "${BASE_DIR}/domainCheck/.venv/bin/python"
+require_path "${BASE_DIR}/domainCheck/detect_worker.py"
+require_path "${BASE_DIR}/domain-api"
+
+echo "[2/6] make release root writable for ${WWW_USER}:${WWW_GROUP}"
+chgrp "${WWW_GROUP}" "${BASE_DIR}"
+chmod 2775 "${BASE_DIR}"
+mkdir -p "${BASE_DIR}/downloads" "${BASE_DIR}/releases"
+chown -R "${WWW_USER}:${WWW_GROUP}" "${BASE_DIR}/downloads" "${BASE_DIR}/releases"
+
+echo "[3/6] ensure current link exists"
+if [[ -L "${BASE_DIR}/current" ]]; then
+  echo "keep existing symlink: ${BASE_DIR}/current -> $(readlink -f "${BASE_DIR}/current" || true)"
+elif [[ -e "${BASE_DIR}/current" ]]; then
+  echo "path exists but is not a symlink: ${BASE_DIR}/current" >&2
+  exit 1
+else
+  ln -s "${BASE_DIR}" "${BASE_DIR}/current"
+  echo "created symlink: ${BASE_DIR}/current -> ${BASE_DIR}"
+fi
+
+echo "[4/6] install systemd drop-ins"
+install_dropin "domaincheck-worker" "[Service]
+WorkingDirectory=${BASE_DIR}/current/domainCheck
+ExecStart=
+ExecStart=${BASE_DIR}/domainCheck/.venv/bin/python ${BASE_DIR}/current/domainCheck/detect_worker.py"
+
+install_dropin "domaincheck-node-agent" "[Service]
+User=root
+Group=root
+WorkingDirectory=${BASE_DIR}/current/domain-api
+ExecStart=
+ExecStart=${BASE_DIR}/domainCheck/.venv/bin/python -m app.node_agent"
+
+if [[ "${ROLE}" == "control" ]]; then
+  install_dropin "domaincheck-api" "[Service]
+WorkingDirectory=${BASE_DIR}/current/domain-api
+ExecStart=
+ExecStart=${BASE_DIR}/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100"
+
+  install_dropin "domaincheck-sync-agent" "[Service]
+WorkingDirectory=${BASE_DIR}/current/domain-api
+ExecStart=
+ExecStart=${BASE_DIR}/domainCheck/.venv/bin/python -m app.sync_agent"
+fi
+
+echo "[5/6] reload systemd and restart node agent"
+systemctl daemon-reload
+systemctl restart domaincheck-node-agent
+
+echo "[6/6] summary"
+echo "role=${ROLE}"
+echo "base_dir=${BASE_DIR}"
+echo "current_target=$(readlink -f "${BASE_DIR}/current" || true)"
+echo
+echo "recommended next checks:"
+echo "  systemctl status domaincheck-node-agent --no-pager -l"
+echo "  systemctl cat domaincheck-worker"
+if [[ "${ROLE}" == "control" ]]; then
+  echo "  systemctl cat domaincheck-api"
+  echo "  systemctl cat domaincheck-sync-agent"
+fi
--- a/domain-api/deploy/multi-region/install_mainland_controller_quick.sh
+++ b/domain-api/deploy/multi-region/install_mainland_controller_quick.sh
@@ -125,8 +125,8 @@ SYNC_SOURCE_REGION=mainland
 SYNC_TARGET_REGION=overseas
 SYNC_TARGET_API_BASE_URL=${TARGET_API_BASE_URL}
 SYNC_SHARED_TOKEN=${SYNC_SHARED_TOKEN}
-SYNC_BATCH_SIZE=200
-SYNC_POLL_INTERVAL_SECONDS=30
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
 EOF

 systemctl daemon-reload
--- a/domain-api/deploy/multi-region/install_mainland_worker_quick.sh
+++ b/domain-api/deploy/multi-region/install_mainland_worker_quick.sh
@@ -126,8 +126,8 @@ SYNC_SOURCE_REGION=mainland
 SYNC_TARGET_REGION=overseas
 SYNC_TARGET_API_BASE_URL=${TARGET_API_BASE_URL}
 SYNC_SHARED_TOKEN=${SYNC_SHARED_TOKEN}
-SYNC_BATCH_SIZE=200
-SYNC_POLL_INTERVAL_SECONDS=30
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
 EOF

 systemctl daemon-reload
--- a/domain-api/deploy/multi-region/night_shift_until_20260420_1200.sh
+++ b/domain-api/deploy/multi-region/night_shift_until_20260420_1200.sh
@@ -0,0 +1,303 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+BASE_URL="${1:-http://127.0.0.1:8100}"
+DEADLINE_TEXT="${2:-2026-04-20 12:00:00 +0800}"
+RUNTIME_DIR="${REPO_ROOT}/docs/ops_center_runtime/night_runs"
+START_TS="$(date +%Y%m%d_%H%M%S)"
+RUN_ID="night_run_${START_TS}"
+LOG_FILE="${RUNTIME_DIR}/${RUN_ID}.log"
+STATE_DIR="${RUNTIME_DIR}/${RUN_ID}"
+PID_FILE="${RUNTIME_DIR}/${RUN_ID}.pid"
+REPORT_FILE="${RUNTIME_DIR}/${RUN_ID}_report.md"
+SUMMARY_JSON="${RUNTIME_DIR}/${RUN_ID}_summary.json"
+
+mkdir -p "${RUNTIME_DIR}" "${STATE_DIR}"
+echo "$$" > "${PID_FILE}"
+
+DEADLINE_EPOCH="$(python3 - <<'PY' "${DEADLINE_TEXT}"
+import datetime
+import sys
+
+text = sys.argv[1]
+dt = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M:%S %z")
+print(int(dt.timestamp()))
+PY
+)"
+
+cycles=0
+log_sync_recover_runs=0
+inspection_runs=0
+inspection_churn_runs=0
+quick_rechecks=0
+stop_reason="deadline_reached"
+last_go_live_status=""
+last_publish_ready=""
+last_log_sync_state=""
+last_issue_total=""
+last_problem_runs_total=""
+last_launchpad_status=""
+last_launchpad_action=""
+last_problem_run_code=""
+
+log() {
+  printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S %z')" "$*" | tee -a "${LOG_FILE}"
+}
+
+run_cmd() {
+  log "RUN $*"
+  if "$@" >> "${LOG_FILE}" 2>&1; then
+    log "OK  $*"
+    return 0
+  fi
+  log "ERR $*"
+  return 1
+}
+
+fetch_json() {
+  local url="$1"
+  local target="$2"
+  if curl -m 20 -s "${url}" > "${target}.tmp"; then
+    mv "${target}.tmp" "${target}"
+    return 0
+  fi
+  rm -f "${target}.tmp"
+  return 1
+}
+
+snapshot_state() {
+  local cycle_dir="$1"
+  mkdir -p "${cycle_dir}"
+
+  fetch_json "${BASE_URL}/api/v1/ops/go-live-summary" "${cycle_dir}/go_live.json" || true
+  fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis" "${cycle_dir}/stack.json" || true
+  fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" "${cycle_dir}/launchpad.json" || true
+  fetch_json "${BASE_URL}/api/v1/ops/playbook-runs" "${cycle_dir}/playbook_runs.json" || true
+  fetch_json "${BASE_URL}/api/v1/ops/nodes/overseas-control-01/scene-log?limit=120&mode=full" "${cycle_dir}/scene_overseas_control_01.json" || true
+
+  python3 - <<'PY' \
+    "${cycle_dir}/go_live.json" \
+    "${cycle_dir}/stack.json" \
+    "${cycle_dir}/launchpad.json" \
+    "${cycle_dir}/playbook_runs.json" \
+    "${cycle_dir}/scene_overseas_control_01.json" \
+    "${SUMMARY_JSON}" \
+    "${cycles}" \
+    > "${cycle_dir}/summary.env"
+import json
+import os
+import sys
+
+go_path, stack_path, launch_path, runs_path, scene_path, summary_path, cycles = sys.argv[1:8]
+
+def load_json(path):
+    try:
+        with open(path, "r", encoding="utf-8") as fh:
+            return json.load(fh)
+    except Exception:
+        return {}
+
+go_data = load_json(go_path).get("data", {})
+stack_data = load_json(stack_path).get("data", {}).get("diagnosis", {})
+launch_data = load_json(launch_path).get("data", {})
+runs_data = load_json(runs_path).get("data", {})
+scene_data = load_json(scene_path).get("data", {})
+
+issues = stack_data.get("issues") or []
+issue_codes = [str(item.get("code") or "") for item in issues if item.get("code")]
+problem_runs = runs_data.get("problem_runs") or []
+problem_run_code = ""
+if problem_runs:
+    problem_run_code = str(problem_runs[0].get("run_code") or "")
+
+summary = {
+    "cycles": int(cycles),
+    "go_live_status": str(go_data.get("go_live_status") or ""),
+    "publish_ready": bool(go_data.get("publish_ready")),
+    "log_sync_state": str(go_data.get("log_sync_state") or ""),
+    "log_sync_missing_node_codes": go_data.get("log_sync_missing_node_codes") or [],
+    "stack_status": str(stack_data.get("stack_status") or ""),
+    "issue_total": int(stack_data.get("issue_total") or 0),
+    "blocking_issue_total": int(stack_data.get("blocking_issue_total") or 0),
+    "issue_codes": issue_codes,
+    "launchpad_status": str((launch_data.get("launchpad_status") or {}).get("status") or ""),
+    "launchpad_recommended_action": str((launch_data.get("launchpad_status") or {}).get("recommended_action_code") or ""),
+    "problem_runs_total": int(runs_data.get("problem_runs_total") or 0),
+    "problem_run_code": problem_run_code,
+    "scene_status": str(scene_data.get("status") or ""),
+    "scene_line_count": int((scene_data.get("source_summary") or {}).get("line_count") or 0),
+    "generated_at": str(go_data.get("generated_at") or stack_data.get("generated_at") or ""),
+}
+
+with open(summary_path, "w", encoding="utf-8") as fh:
+    json.dump(summary, fh, ensure_ascii=False, indent=2)
+
+def emit(key, value):
+    if isinstance(value, bool):
+        value = "true" if value else "false"
+    elif isinstance(value, list):
+        value = ",".join(str(item) for item in value)
+    else:
+        value = str(value)
+    print(f'{key}="{value}"')
+
+for key, value in summary.items():
+    emit(key.upper(), value)
+PY
+
+  # shellcheck disable=SC1090
+  source "${cycle_dir}/summary.env"
+  last_go_live_status="${GO_LIVE_STATUS}"
+  last_publish_ready="${PUBLISH_READY}"
+  last_log_sync_state="${LOG_SYNC_STATE}"
+  last_issue_total="${ISSUE_TOTAL}"
+  last_problem_runs_total="${PROBLEM_RUNS_TOTAL}"
+  last_launchpad_status="${LAUNCHPAD_STATUS}"
+  last_launchpad_action="${LAUNCHPAD_RECOMMENDED_ACTION}"
+  last_problem_run_code="${PROBLEM_RUN_CODE}"
+}
+
+write_report() {
+  python3 - <<'PY' \
+    "${REPORT_FILE}" \
+    "${RUN_ID}" \
+    "${BASE_URL}" \
+    "${DEADLINE_TEXT}" \
+    "${stop_reason}" \
+    "${cycles}" \
+    "${log_sync_recover_runs}" \
+    "${inspection_runs}" \
+    "${inspection_churn_runs}" \
+    "${quick_rechecks}" \
+    "${last_go_live_status}" \
+    "${last_publish_ready}" \
+    "${last_log_sync_state}" \
+    "${last_issue_total}" \
+    "${last_problem_runs_total}" \
+    "${last_launchpad_status}" \
+    "${last_launchpad_action}" \
+    "${last_problem_run_code}" \
+    "${SUMMARY_JSON}"
+import json
+import sys
+
+(
+    report_path,
+    run_id,
+    base_url,
+    deadline_text,
+    stop_reason,
+    cycles,
+    log_sync_recover_runs,
+    inspection_runs,
+    inspection_churn_runs,
+    quick_rechecks,
+    last_go_live_status,
+    last_publish_ready,
+    last_log_sync_state,
+    last_issue_total,
+    last_problem_runs_total,
+    last_launchpad_status,
+    last_launchpad_action,
+    last_problem_run_code,
+    summary_json_path,
+) = sys.argv[1:20]
+
+summary = {}
+try:
+    with open(summary_json_path, "r", encoding="utf-8") as fh:
+        summary = json.load(fh)
+except Exception:
+    summary = {}
+
+lines = [
+    f"# NIGHT RUN REPORT {run_id}",
+    "",
+    f"- Base URL: `{base_url}`",
+    f"- Deadline: `{deadline_text}`",
+    f"- Stop Reason: `{stop_reason}`",
+    f"- Cycles: `{cycles}`",
+    f"- Log Sync Recover Runs: `{log_sync_recover_runs}`",
+    f"- Inspection Runs: `{inspection_runs}`",
+    f"- Inspection Churn Runs: `{inspection_churn_runs}`",
+    f"- Quick Rechecks: `{quick_rechecks}`",
+    "",
+    "## Final Snapshot",
+    "",
+    f"- `go_live_status = {last_go_live_status}`",
+    f"- `publish_ready = {last_publish_ready}`",
+    f"- `log_sync_state = {last_log_sync_state}`",
+    f"- `issue_total = {last_issue_total}`",
+    f"- `problem_runs_total = {last_problem_runs_total}`",
+    f"- `launchpad_status = {last_launchpad_status}`",
+    f"- `launchpad_recommended_action = {last_launchpad_action}`",
+    f"- `problem_run_code = {last_problem_run_code}`",
+    "",
+    "## Summary JSON",
+    "",
+    "```json",
+    json.dumps(summary, ensure_ascii=False, indent=2),
+    "```",
+]
+
+with open(report_path, "w", encoding="utf-8") as fh:
+    fh.write("\n".join(lines) + "\n")
+PY
+}
+
+cleanup() {
+  write_report
+  log "night run stopped: reason=${stop_reason}"
+  log "report: ${REPORT_FILE}"
+  rm -f "${PID_FILE}"
+}
+
+trap cleanup EXIT
+
+log "night run started: run_id=${RUN_ID}"
+log "base_url=${BASE_URL}"
+log "deadline=${DEADLINE_TEXT}"
+log "state_dir=${STATE_DIR}"
+
+while true; do
+  now_epoch="$(date +%s)"
+  if [[ "${now_epoch}" -ge "${DEADLINE_EPOCH}" ]]; then
+    stop_reason="deadline_reached"
+    break
+  fi
+
+  cycles=$((cycles + 1))
+  cycle_dir="${STATE_DIR}/cycle_${cycles}"
+  snapshot_state "${cycle_dir}"
+
+  log "cycle=${cycles} go_live=${last_go_live_status} log_sync=${last_log_sync_state} issue_total=${last_issue_total} problem_runs=${last_problem_runs_total} launchpad=${last_launchpad_status}/${last_launchpad_action} focus_run=${last_problem_run_code}"
+
+  if [[ "${last_log_sync_state}" != "full_capture" ]]; then
+    log "action: recover log sync coverage"
+    run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" log-sync-recover "${BASE_URL}" full confirm cli/night-pack || true
+    log_sync_recover_runs=$((log_sync_recover_runs + 1))
+    run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true
+    inspection_runs=$((inspection_runs + 1))
+    quick_rechecks=$((quick_rechecks + 1))
+    sleep 45
+    continue
+  fi
+
+  if [[ "${last_issue_total}" != "0" && "${last_problem_runs_total}" != "0" && "${inspection_churn_runs}" -lt 3 ]]; then
+    log "action: create fresh safe inspection run to dilute stale problem run window"
+    run_cmd bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-run "${BASE_URL}" run_inspection_participating '{}' confirm cli/night-pack || true
+    inspection_runs=$((inspection_runs + 1))
+    inspection_churn_runs=$((inspection_churn_runs + 1))
+    sleep 1800
+    continue
+  fi
+
+  if [[ "${last_issue_total}" == "0" && "${last_log_sync_state}" == "full_capture" && "${last_problem_runs_total}" == "0" ]]; then
+    stop_reason="signoff_ready_candidate"
+    break
+  fi
+
+  sleep 3600
+done
--- a/domain-api/deploy/multi-region/templates/domaincheck-api.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-api.env.example
@@ -26,5 +26,5 @@ SYNC_SOURCE_REGION=overseas
 SYNC_TARGET_REGION=overseas
 SYNC_TARGET_API_BASE_URL=
 SYNC_SHARED_TOKEN=
-SYNC_BATCH_SIZE=200
-SYNC_POLL_INTERVAL_SECONDS=30
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
--- a/domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example
@@ -11,5 +11,5 @@ API_SERVICE_NAME=domaincheck-api
 SYNC_AGENT_SERVICE_NAME=domaincheck-sync-agent
 NODE_AGENT_SERVICE_NAME=domaincheck-node-agent

-OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.start_detection","runtime.stop_detection","runtime.pull_tasks","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]
+OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.start_detection","runtime.stop_detection","runtime.pull_tasks","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","runtime.reset_lab_state","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]
 OPS_AGENT_LABELS={}
--- a/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example
@@ -23,5 +23,5 @@ SYNC_SOURCE_REGION=mainland
 SYNC_TARGET_REGION=overseas
 SYNC_TARGET_API_BASE_URL=http://海外控制面IP:8100/api/v1
 SYNC_SHARED_TOKEN=
-SYNC_BATCH_SIZE=200
-SYNC_POLL_INTERVAL_SECONDS=30
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
--- a/domain-api/deploy/multi-region/templates/domaincheck-worker.worker.env.example
+++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.worker.env.example
@@ -23,5 +23,5 @@ SYNC_SOURCE_REGION=mainland
 SYNC_TARGET_REGION=overseas
 SYNC_TARGET_API_BASE_URL=http://海外控制面IP:8100/api/v1
 SYNC_SHARED_TOKEN=
-SYNC_BATCH_SIZE=200
-SYNC_POLL_INTERVAL_SECONDS=30
+SYNC_BATCH_SIZE=5000
+SYNC_POLL_INTERVAL_SECONDS=2
--- a/domain-api/deploy/systemd/domain-node-agent.service
+++ b/domain-api/deploy/systemd/domain-node-agent.service
@@ -6,13 +6,14 @@ After=network.target
 Type=simple
 WorkingDirectory=/opt/domaincheck/domain-api
 EnvironmentFile=-/etc/default/domaincheck-api
+EnvironmentFile=-/etc/default/domaincheck-worker
 EnvironmentFile=-/etc/default/domaincheck-node-agent
 Environment="PATH=/home/www/.nvm/versions/node/v20.20.2/bin:/home/www/.local/bin:/home/www/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
 ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m app.node_agent
 Restart=always
 RestartSec=5
-User=www
-Group=www
+User=root
+Group=root
 SupplementaryGroups=systemd-journal

 [Install]