feat: stabilize multi-region runtime sync and worker orchestration

tools/collect_chinaz_gray_metrics.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+
+def run_ssh(remote_cmd: str, timeout: int) -> str:
+    proc = subprocess.run(
+        [
+            "python3",
+            "-c",
+            (
+                "import paramiko,sys;"
+                "host,user,password,cmd,timeout=sys.argv[1:6];"
+                "client=paramiko.SSHClient();"
+                "client.set_missing_host_key_policy(paramiko.AutoAddPolicy());"
+                "client.connect(host, username=user, password=password, timeout=int(timeout));"
+                "stdin,stdout,stderr=client.exec_command(cmd, timeout=int(timeout));"
+                "sys.stdout.write(stdout.read().decode('utf-8','ignore'));"
+                "err=stderr.read().decode('utf-8','ignore');"
+                "client.close();"
+                "sys.stderr.write(err)"
+            ),
+            os.environ["REMOTE_HOST"],
+            os.environ["REMOTE_USER"],
+            os.environ["REMOTE_PASS"],
+            remote_cmd,
+            str(timeout),
+        ],
+        capture_output=True,
+        text=True,
+        timeout=timeout + 10,
+    )
+    if proc.returncode != 0 and proc.stderr.strip():
+        raise RuntimeError(proc.stderr.strip())
+    return proc.stdout
+
+
+def parse_cpu_busy(text: str) -> Optional[float]:
+    match = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*id", text)
+    if not match:
+        return None
+    idle = float(match.group(1))
+    return round(max(0.0, 100.0 - idle), 2)
+
+
+def collect_snapshot(gray_workers: List[str], since: str) -> dict:
+    units = " ".join(f"-u domaincheck-worker@{worker}.service" for worker in gray_workers)
+    remote_cmd = (
+        "echo '###CPU'; "
+        "top -bn1 | grep '%Cpu' || true; "
+        "echo '###GRAY'; "
+        f"journalctl {units} --since '{since}' --no-pager "
+        "| grep -E 'proxy_direct_fallback|站长之家调用边界|single_step_finalized|重试预算耗尽' "
+        "| tail -n 1200 || true; "
+        "echo '###GLOBAL'; "
+        "journalctl -u 'domaincheck-worker@*.service' "
+        f"--since '{since}' --no-pager "
+        "| grep 'detect_order=detect_chinaz' | tail -n 1200 || true"
+    )
+    out = run_ssh(remote_cmd, timeout=40)
+    cpu_part = ""
+    gray_part = ""
+    global_part = ""
+    if "###GRAY" in out:
+        cpu_part, rest = out.split("###GRAY", 1)
+        if "###GLOBAL" in rest:
+            gray_part, global_part = rest.split("###GLOBAL", 1)
+        else:
+            gray_part = rest
+    else:
+        cpu_part = out
+    cpu_busy = parse_cpu_busy(cpu_part)
+
+    counts = {
+        "proxy_direct_fallback": 0,
+        "boundary_proxy_yes": 0,
+        "boundary_proxy_no": 0,
+        "single_step_finalized": 0,
+        "retry_budget_exhausted": 0,
+    }
+    per_worker: Dict[str, Dict[str, int]] = {}
+    for line in gray_part.splitlines():
+        worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
+        worker = worker_match.group(1) if worker_match else "-"
+        bucket = per_worker.setdefault(
+            worker,
+            {
+                "proxy_direct_fallback": 0,
+                "boundary_proxy_yes": 0,
+                "boundary_proxy_no": 0,
+                "single_step_finalized": 0,
+                "retry_budget_exhausted": 0,
+            },
+        )
+        if "proxy_direct_fallback" in line:
+            counts["proxy_direct_fallback"] += 1
+            bucket["proxy_direct_fallback"] += 1
+        if "站长之家调用边界" in line and "proxy=yes" in line:
+            counts["boundary_proxy_yes"] += 1
+            bucket["boundary_proxy_yes"] += 1
+        if "站长之家调用边界" in line and "proxy=no" in line:
+            counts["boundary_proxy_no"] += 1
+            bucket["boundary_proxy_no"] += 1
+        if "single_step_finalized" in line and "detect_key=detect_chinaz" in line:
+            counts["single_step_finalized"] += 1
+            bucket["single_step_finalized"] += 1
+        if "重试预算耗尽" in line and "站长之家检测" in line:
+            counts["retry_budget_exhausted"] += 1
+            bucket["retry_budget_exhausted"] += 1
+
+    global_worker_counts: Dict[str, int] = {}
+    if global_part.strip():
+        top_counts: Dict[str, int] = {}
+        for line in global_part.splitlines():
+            worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
+            if not worker_match:
+                continue
+            worker = worker_match.group(1)
+            top_counts[worker] = top_counts.get(worker, 0) + 1
+        global_worker_counts = dict(
+            sorted(top_counts.items(), key=lambda item: item[1], reverse=True)[:12]
+        )
+
+    return {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "cpu_busy_pct": cpu_busy,
+        "counts": counts,
+        "per_worker": per_worker,
+        "global_detect_chinaz_orders": sum(global_worker_counts.values()),
+        "global_detect_chinaz_top_workers": global_worker_counts,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--workers", required=True, help="comma-separated worker suffixes")
+    parser.add_argument("--duration-seconds", type=int, default=3600)
+    parser.add_argument("--interval-seconds", type=int, default=30)
+    parser.add_argument("--since-window-seconds", type=int, default=45)
+    args = parser.parse_args()
+
+    gray_workers = [item.strip() for item in args.workers.split(",") if item.strip()]
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+    deadline = time.time() + max(0, args.duration_seconds)
+    while time.time() <= deadline:
+        since_ts = datetime.fromtimestamp(
+            time.time() - max(1, args.since_window_seconds),
+            tz=timezone.utc,
+        ).astimezone().strftime("%Y-%m-%d %H:%M:%S")
+        snapshot = collect_snapshot(gray_workers, since_ts)
+        with open(args.output, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(snapshot, ensure_ascii=False) + "\n")
+        time.sleep(max(1, args.interval_seconds))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/collect_overnight_metrics.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import paramiko
+import psycopg2
+import requests
+
+
+def _now() -> datetime:
+    return datetime.now()
+
+
+def _safe_json_response(url: str) -> dict:
+    try:
+        response = requests.get(url, timeout=8)
+        payload = response.json()
+        if isinstance(payload, dict):
+            return payload.get("data") or payload
+    except Exception as exc:
+        return {"error": str(exc), "url": url}
+    return {}
+
+
+def _query_local_db() -> dict:
+    try:
+        conn = psycopg2.connect(
+            host="127.0.0.1",
+            port=5432,
+            dbname="domain",
+            user="postgres",
+            password="Qazwe123,./",
+        )
+        with conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    """
+                    select node_code, status, current_load, last_heartbeat_at
+                    from detect_worker_nodes
+                    where node_code in ('mainland-controller-01', 'mainland-worker-01')
+                    order by node_code
+                    """
+                )
+                worker_nodes = [
+                    {
+                        "node_code": row[0],
+                        "status": row[1],
+                        "current_load": int(row[2] or 0),
+                        "last_heartbeat_at": row[3].isoformat(sep=" ", timespec="seconds") if row[3] else "",
+                    }
+                    for row in cur.fetchall()
+                ]
+        conn.close()
+        return {"worker_nodes": worker_nodes}
+    except Exception as exc:
+        return {"error": str(exc)}
+
+
+def _run_remote_sample(host: str, user: str, password: str) -> dict:
+    if not str(host or "").strip() or not str(user or "").strip() or not str(password or "").strip():
+        return {"skipped": True, "reason": "remote ssh credentials not provided"}
+    cli = paramiko.SSHClient()
+    cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        cli.connect(hostname=host, username=user, password=password, timeout=10)
+        command = r"""printf 'proc='; pgrep -fc '[d]etect_worker.py'
+printf 'load='; ps -eo pcpu,cmd | grep '[d]etect_worker.py' | awk '{s+=$1} END {print s+0}'
+printf 'db_err_recent='; journalctl -u domaincheck-worker -u 'domaincheck-worker@*' --since '5 min ago' --no-pager | egrep -c '初始化数据库连接池失败|连接池耗尽|等待超时|数据库连接失败' || true
+cd /www/wwwroot/getDomain/domain-api && /opt/domaincheck/domainCheck/.venv/bin/python - <<'INNER'
+import json
+from app.services.detect_service import get_detect_status
+result = get_detect_status() or {}
+print('detect_status=' + json.dumps({
+    'job_code': result.get('job_code'),
+    'active_thread_count': result.get('active_thread_count'),
+    'max_thread_count': result.get('max_thread_count'),
+    'aggregate_process_count': result.get('aggregate_process_count'),
+    'display_items_running': result.get('display_items_running'),
+    'display_active_threads': result.get('display_active_threads'),
+    'display_max_threads': result.get('display_max_threads'),
+}, ensure_ascii=False))
+INNER"""
+        _, stdout, stderr = cli.exec_command(command, timeout=45)
+        output = stdout.read().decode("utf-8", errors="replace")
+        error = stderr.read().decode("utf-8", errors="replace").strip()
+        sample: dict[str, object] = {"raw": output.strip()}
+        if error:
+            sample["stderr"] = error
+        for line in output.splitlines():
+            if "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            if key == "detect_status":
+                try:
+                    sample[key] = json.loads(value)
+                except Exception:
+                    sample[key] = value
+                continue
+            try:
+                sample[key] = int(value)
+            except Exception:
+                try:
+                    sample[key] = float(value)
+                except Exception:
+                    sample[key] = value
+        return sample
+    except Exception as exc:
+        return {"error": str(exc)}
+    finally:
+        try:
+            cli.close()
+        except Exception:
+            pass
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Collect overnight worker metrics into JSONL.")
+    parser.add_argument("--worker-host", default="")
+    parser.add_argument("--worker-user", default="")
+    parser.add_argument("--worker-password", default="")
+    parser.add_argument("--interval-seconds", type=int, default=60)
+    parser.add_argument("--duration-hours", type=float, default=6.0)
+    parser.add_argument("--label", default="mainland-worker-01-night")
+    args = parser.parse_args()
+
+    start_time = _now()
+    end_time = start_time + timedelta(hours=max(0.25, float(args.duration_hours)))
+    run_dir = Path("/www/wwwroot/getDomain/docs/ops_center_runtime/night_runs") / (
+        f"night_run_{start_time.strftime('%Y%m%d_%H%M%S')}_{args.label}"
+    )
+    run_dir.mkdir(parents=True, exist_ok=True)
+    metrics_path = run_dir / "metrics.jsonl"
+    meta_path = run_dir / "meta.json"
+    meta_path.write_text(
+        json.dumps(
+            {
+                "label": args.label,
+                "started_at": start_time.isoformat(sep=" ", timespec="seconds"),
+                "planned_end_at": end_time.isoformat(sep=" ", timespec="seconds"),
+                "interval_seconds": int(args.interval_seconds),
+                "worker_host": args.worker_host,
+                "remote_sampling_enabled": bool(
+                    str(args.worker_host or "").strip()
+                    and str(args.worker_user or "").strip()
+                    and str(args.worker_password or "").strip()
+                ),
+            },
+            ensure_ascii=False,
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+
+    base_url = "http://127.0.0.1:8100/api/v1"
+    sample_index = 0
+    while _now() < end_time:
+        sample_index += 1
+        timestamp = _now().isoformat(sep=" ", timespec="seconds")
+        row = {
+            "timestamp": timestamp,
+            "sample_index": sample_index,
+            "local": {
+                "detect_status": _safe_json_response(f"{base_url}/detect/status"),
+                "runtime_status": _safe_json_response(f"{base_url}/runtime/status"),
+                "dashboard_overview": _safe_json_response(f"{base_url}/dashboard/overview"),
+                "db": _query_local_db(),
+            },
+            "remote_worker": _run_remote_sample(
+                host=args.worker_host,
+                user=args.worker_user,
+                password=args.worker_password,
+            ),
+        }
+        with metrics_path.open("a", encoding="utf-8") as fp:
+            fp.write(json.dumps(row, ensure_ascii=False) + "\n")
+        time.sleep(max(10, int(args.interval_seconds)))
+
+    (run_dir / "done.flag").write_text(_now().isoformat(sep=" ", timespec="seconds"), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/collect_remote_step_mix.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import paramiko
+
+
+def _now() -> datetime:
+    return datetime.now()
+
+
+def _parse_key_value_lines(raw: str) -> dict:
+    result: dict[str, object] = {"raw": raw.strip()}
+    for line in raw.splitlines():
+        if "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        key = key.strip()
+        value = value.strip()
+        if not key:
+            continue
+        if key in {"running_jobs", "step_counts"}:
+            try:
+                result[key] = json.loads(value)
+            except Exception:
+                result[key] = value
+            continue
+        for caster in (int, float):
+            try:
+                result[key] = caster(value)
+                break
+            except Exception:
+                continue
+        else:
+            result[key] = value
+    return result
+
+
+def _remote_sample(host: str, user: str, password: str, recent_minutes: int) -> dict:
+    cli = paramiko.SSHClient()
+    cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    command = """
+python3 - <<'PY'
+import json, re, subprocess
+
+def sh(cmd: str) -> str:
+    return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL)
+
+cpu_line = sh("top -bn1 | sed -n '3p'").strip()
+idle_match = re.search(r'([0-9]+(?:\.[0-9]+)?)\s+id', cpu_line)
+busy = round(100.0 - float(idle_match.group(1)), 2) if idle_match else None
+print(f"cpu_busy_pct={busy if busy is not None else ''}")
+
+worker_stats = sh(r'''ps -eo pcpu,cmd | awk '/[d]etect_worker.py/ {cpu=$1+0; total++; sum+=cpu; if(cpu>1) gt1++; if(cpu>2) gt2++; if(cpu>3) gt3++;} END {printf("workers_total=%d\\nworkers_gt_1pct=%d\\nworkers_gt_2pct=%d\\nworkers_gt_3pct=%d\\nworkers_cpu_sum=%.1f\\n", total, gt1, gt2, gt3, sum)}' ''')
+print(worker_stats.strip())
+
+running_jobs_raw = sh(r'''/www/server/pgsql/bin/psql -U postgres -d domain -At -c "select id, job_code, status from detect_jobs where status='running' order by id desc limit 16;" ''')
+running_jobs = []
+for line in running_jobs_raw.splitlines():
+    parts = line.split("|")
+    if len(parts) >= 3:
+        running_jobs.append({"job_id": int(parts[0]), "job_code": parts[1], "status": parts[2]})
+print("running_jobs=" + json.dumps(running_jobs, ensure_ascii=False))
+
+recent = __RECENT_MINUTES__
+journal_cmd = f"journalctl -u domaincheck-worker -u 'domaincheck-worker@*' --since '{recent} min ago' --no-pager -o cat"
+logs = sh(journal_cmd)
+step_counts = {}
+for match in re.finditer(r'detect_order=(detect_[a-zA-Z0-9_]+)', logs):
+    step = match.group(1)
+    step_counts[step] = step_counts.get(step, 0) + 1
+print("step_counts=" + json.dumps(step_counts, ensure_ascii=False, sort_keys=True))
+PY
+"""
+    command = command.replace("__RECENT_MINUTES__", str(int(recent_minutes)))
+    try:
+        cli.connect(hostname=host, username=user, password=password, timeout=15, banner_timeout=15, auth_timeout=15)
+        _, stdout, stderr = cli.exec_command(command, timeout=90)
+        output = stdout.read().decode("utf-8", errors="replace")
+        error = stderr.read().decode("utf-8", errors="replace").strip()
+        row = _parse_key_value_lines(output)
+        if error:
+            row["stderr"] = error
+        return row
+    except Exception as exc:
+        return {"error": str(exc)}
+    finally:
+        try:
+            cli.close()
+        except Exception:
+            pass
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Collect remote CPU/worker/step-mix metrics into JSONL.")
+    parser.add_argument("--worker-host", required=True)
+    parser.add_argument("--worker-user", required=True)
+    parser.add_argument("--worker-password", required=True)
+    parser.add_argument("--interval-seconds", type=int, default=120)
+    parser.add_argument("--duration-hours", type=float, default=8.0)
+    parser.add_argument("--recent-minutes", type=int, default=3)
+    parser.add_argument("--label", default="remote-step-mix")
+    args = parser.parse_args()
+
+    start_time = _now()
+    end_time = start_time + timedelta(hours=max(0.25, float(args.duration_hours)))
+    run_dir = Path("/www/wwwroot/getDomain/docs/ops_center_runtime/night_runs") / (
+        f"step_mix_{start_time.strftime('%Y%m%d_%H%M%S')}_{args.label}"
+    )
+    run_dir.mkdir(parents=True, exist_ok=True)
+    metrics_path = run_dir / "metrics.jsonl"
+    meta_path = run_dir / "meta.json"
+    meta_path.write_text(
+        json.dumps(
+            {
+                "label": args.label,
+                "started_at": start_time.isoformat(sep=" ", timespec="seconds"),
+                "planned_end_at": end_time.isoformat(sep=" ", timespec="seconds"),
+                "interval_seconds": int(args.interval_seconds),
+                "recent_minutes": int(args.recent_minutes),
+                "worker_host": args.worker_host,
+            },
+            ensure_ascii=False,
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+
+    sample_index = 0
+    while _now() < end_time:
+        sample_index += 1
+        row = {
+            "timestamp": _now().isoformat(sep=" ", timespec="seconds"),
+            "sample_index": sample_index,
+            "remote_worker": _remote_sample(
+                host=args.worker_host,
+                user=args.worker_user,
+                password=args.worker_password,
+                recent_minutes=max(1, int(args.recent_minutes)),
+            ),
+        }
+        with metrics_path.open("a", encoding="utf-8") as fp:
+            fp.write(json.dumps(row, ensure_ascii=False) + "\n")
+        time.sleep(max(30, int(args.interval_seconds)))
+
+    (run_dir / "done.flag").write_text(_now().isoformat(sep=" ", timespec="seconds"), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/quiet_window_rebuild_release_index.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""Quiet-window helper for rebuilding idx_detect_job_items_release_node_job.
+
+Stops node-agent and worker services, keeps clearing blockers for the release-index
+DROP INDEX backend until it disappears, waits briefly for CREATE INDEX to appear,
+then restores services.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DOMAINCHECK_ROOT = ROOT / "domainCheck"
+if str(DOMAINCHECK_ROOT) not in sys.path:
+    sys.path.insert(0, str(DOMAINCHECK_ROOT))
+
+from app.utils.database import Database  # noqa: E402
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_release_node_job"
+POLL_SECONDS = 5
+MAX_DROP_LOOPS = 72  # about 6 minutes
+MAX_RESTORE_LOOPS = 24
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def run(cmd: str) -> tuple[int, str, str]:
+    proc = subprocess.run(
+        cmd,
+        shell=True,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    return proc.returncode, proc.stdout.strip(), proc.stderr.strip()
+
+
+def service_state(name: str) -> str:
+    _, out, _ = run(f"systemctl is-active {name} || true")
+    return out.strip() or "unknown"
+
+
+def templated_running_count() -> int:
+    _, out, _ = run(
+        "systemctl list-units --type=service --state=running 'domaincheck-worker@*.service' --no-legend | wc -l"
+    )
+    try:
+        return int((out or "0").strip())
+    except Exception:
+        return -1
+
+
+def stop_services() -> None:
+    for cmd in [
+        "systemctl stop domaincheck-node-agent.service",
+        "systemctl stop 'domaincheck-worker@*.service'",
+        "systemctl stop domaincheck-worker.service",
+    ]:
+        code, out, err = run(cmd)
+        log(f"stop cmd={cmd!r} code={code} out={out!r} err={err!r}")
+
+
+def start_services() -> None:
+    for cmd in [
+        "systemctl start domaincheck-node-agent.service",
+        "systemctl start domaincheck-worker.service",
+        "systemctl start --all 'domaincheck-worker@*.service'",
+    ]:
+        code, out, err = run(cmd)
+        log(f"start cmd={cmd!r} code={code} out={out!r} err={err!r}")
+
+
+def wait_for_quiet() -> None:
+    for loop in range(MAX_RESTORE_LOOPS):
+        templated = templated_running_count()
+        base = service_state("domaincheck-worker.service")
+        node_agent = service_state("domaincheck-node-agent.service")
+        log(
+            f"quiet_check loop={loop} templated={templated} "
+            f"base={base} node_agent={node_agent}"
+        )
+        if templated == 0 and base != "active" and node_agent != "active":
+            return
+        time.sleep(POLL_SECONDS)
+
+
+def wait_for_restore() -> None:
+    for loop in range(MAX_RESTORE_LOOPS):
+        templated = templated_running_count()
+        base = service_state("domaincheck-worker.service")
+        node_agent = service_state("domaincheck-node-agent.service")
+        log(
+            f"restore_check loop={loop} templated={templated} "
+            f"base={base} node_agent={node_agent}"
+        )
+        if templated >= 190 and base == "active" and node_agent == "active":
+            return
+        time.sleep(POLL_SECONDS)
+
+
+def fetch_drop_pids(cur) -> list[int]:
+    cur.execute(
+        """
+        SELECT pid
+        FROM pg_stat_activity
+        WHERE query LIKE %s
+        ORDER BY pid
+        """,
+        (f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",),
+    )
+    return [int(row[0]) for row in cur.fetchall()]
+
+
+def fetch_create_progress(cur) -> list[tuple]:
+    cur.execute(
+        """
+        SELECT pid, phase, blocks_done, blocks_total, current_locker_pid
+        FROM pg_stat_progress_create_index
+        WHERE index_relid = (SELECT oid FROM pg_class WHERE relname = %s)
+        ORDER BY pid
+        """,
+        (INDEX_NAME,),
+    )
+    return list(cur.fetchall())
+
+
+def fetch_index_state(cur) -> str:
+    cur.execute(
+        """
+        SELECT
+            c.relname,
+            i.indisvalid,
+            i.indisready
+        FROM pg_class c
+        JOIN pg_index i ON i.indexrelid = c.oid
+        WHERE c.relname = %s
+        """,
+        (INDEX_NAME,),
+    )
+    row = cur.fetchone()
+    if not row:
+        return "missing"
+    return f"{row[0]}|{int(bool(row[1]))}|{int(bool(row[2]))}"
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    db = Database()
+    conn = db.get_connection()
+    if not conn:
+        log("db_connect_failed")
+        return 1
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        log(f"quiet-window start index={INDEX_NAME}")
+        stop_services()
+        wait_for_quiet()
+
+        for loop in range(MAX_DROP_LOOPS):
+            drop_pids = fetch_drop_pids(cur)
+            log(f"drop_loop={loop} drop_pids={drop_pids or ['none']}")
+            if not drop_pids:
+                break
+            for pid in drop_pids:
+                cur.execute("SELECT pg_blocking_pids(%s)", (pid,))
+                row = cur.fetchone()
+                blockers = [int(v) for v in (row[0] or [])] if row and row[0] else []
+                log(f"drop_pid={pid} blockers={blockers or ['none']}")
+                for blocker in blockers:
+                    cur.execute("SELECT pg_terminate_backend(%s)", (blocker,))
+                    result = cur.fetchone()
+                    log(f"terminate blocker={blocker} result={result!r}")
+            time.sleep(POLL_SECONDS)
+
+        for loop in range(12):
+            progress = fetch_create_progress(cur)
+            state = fetch_index_state(cur)
+            log(f"create_wait loop={loop} index_state={state} progress={progress or ['none']}")
+            if progress:
+                break
+            time.sleep(POLL_SECONDS)
+    finally:
+        try:
+            start_services()
+            wait_for_restore()
+        finally:
+            cur.close()
+            db.close(conn=conn)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/rebuild_release_node_job_index.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Drop and recreate idx_detect_job_items_release_node_job on mainland."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import psycopg2
+from psycopg2 import sql
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_release_node_job"
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def fetch_index_state(cur: psycopg2.extensions.cursor, index_name: str) -> tuple[bool, bool, bool] | None:
+    cur.execute(
+        """
+        SELECT i.indisvalid, i.indisready, i.indislive
+        FROM pg_index i
+        JOIN pg_class c ON c.oid = i.indexrelid
+        WHERE c.relname = %s
+        """,
+        (index_name,),
+    )
+    row = cur.fetchone()
+    if not row:
+        return None
+    return bool(row[0]), bool(row[1]), bool(row[2])
+
+
+def fetch_conflicting_backends(cur: psycopg2.extensions.cursor) -> list[tuple[int, str, str]]:
+    cur.execute(
+        """
+        SELECT pid, query, coalesce(wait_event_type, '')
+        FROM pg_stat_activity
+        WHERE state <> 'idle'
+          AND (
+            query ILIKE %s OR
+            query ILIKE %s
+          )
+        ORDER BY backend_start
+        """,
+        (
+            "%idx_detect_job_items_release_node_job%",
+            "%idx_detect_job_items_stalled_job_activity%",
+        ),
+    )
+    rows = []
+    for pid, query, wait_event_type in cur.fetchall():
+        snippet = " ".join((query or "").split())[:160]
+        rows.append((pid, wait_event_type, snippet))
+    return rows
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    conn = psycopg2.connect(
+        host="127.0.0.1",
+        port=5432,
+        user=os.getenv("DB_USER", "postgres"),
+        password=os.getenv("DB_PASSWORD"),
+        dbname="domain",
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        conflicts = fetch_conflicting_backends(cur)
+        if conflicts:
+            log(f"preflight conflicting_backends={conflicts}")
+
+        log("drop start")
+        cur.execute(sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(sql.Identifier(INDEX_NAME)))
+        log("drop done")
+
+        residual_state = fetch_index_state(cur, INDEX_NAME)
+        if residual_state is not None:
+            raise RuntimeError(
+                f"index shell still present after drop: name={INDEX_NAME} "
+                f"state={residual_state}"
+            )
+
+        log("create start")
+        cur.execute(
+            sql.SQL(
+                """
+                CREATE INDEX CONCURRENTLY {} 
+                ON public.detect_job_items (claimed_by, job_id, status, id)
+                WHERE claimed_by <> ''
+                  AND status IN ('claimed', 'running')
+                """
+            ).format(sql.Identifier(INDEX_NAME))
+        )
+        log("create done")
+        log(f"final state={fetch_index_state(cur, INDEX_NAME)}")
+    finally:
+        cur.close()
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/rebuild_stalled_job_activity_index.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Drop and recreate idx_detect_job_items_stalled_job_activity on mainland."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import psycopg2
+from psycopg2 import sql
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_stalled_job_activity"
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def fetch_index_state(cur: psycopg2.extensions.cursor, index_name: str) -> tuple[bool, bool, bool] | None:
+    cur.execute(
+        """
+        SELECT i.indisvalid, i.indisready, i.indislive
+        FROM pg_index i
+        JOIN pg_class c ON c.oid = i.indexrelid
+        WHERE c.relname = %s
+        """,
+        (index_name,),
+    )
+    row = cur.fetchone()
+    if not row:
+        return None
+    return bool(row[0]), bool(row[1]), bool(row[2])
+
+
+def fetch_conflicting_backends(cur: psycopg2.extensions.cursor) -> list[tuple[int, str, str]]:
+    cur.execute(
+        """
+        SELECT pid, query, coalesce(wait_event_type, '')
+        FROM pg_stat_activity
+        WHERE state <> 'idle'
+          AND (
+            query ILIKE %s OR
+            query ILIKE %s
+          )
+        ORDER BY backend_start
+        """,
+        (
+            "%idx_detect_job_items_stalled_job_activity%",
+            "%idx_detect_job_items_release_node_job%",
+        ),
+    )
+    rows = []
+    for pid, query, wait_event_type in cur.fetchall():
+        snippet = " ".join((query or "").split())[:160]
+        rows.append((pid, wait_event_type, snippet))
+    return rows
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    conn = psycopg2.connect(
+        host="127.0.0.1",
+        port=5432,
+        user=os.getenv("DB_USER", "postgres"),
+        password=os.getenv("DB_PASSWORD"),
+        dbname="domain",
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        conflicts = fetch_conflicting_backends(cur)
+        if conflicts:
+            log(f"preflight conflicting_backends={conflicts}")
+
+        log("drop start")
+        cur.execute(sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(sql.Identifier(INDEX_NAME)))
+        log("drop done")
+
+        residual_state = fetch_index_state(cur, INDEX_NAME)
+        if residual_state is not None:
+            raise RuntimeError(
+                f"index shell still present after drop: name={INDEX_NAME} "
+                f"state={residual_state}"
+            )
+
+        log("create start")
+        cur.execute(
+            sql.SQL(
+                """
+                CREATE INDEX CONCURRENTLY {}
+                ON public.detect_job_items (
+                    job_id,
+                    status,
+                    (COALESCE(updated_at, started_at, create_time)),
+                    id
+                )
+                WHERE status IN ('claimed', 'running')
+                """
+            ).format(sql.Identifier(INDEX_NAME))
+        )
+        log("create done")
+        log(f"final state={fetch_index_state(cur, INDEX_NAME)}")
+    finally:
+        cur.close()
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/runtime_observer.py

@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+
+
+DEFAULT_API_BASE = "http://127.0.0.1:8100/api/v1"
+
+
+def _now_text() -> str:
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _safe_int(value: object, default: int = 0) -> int:
+    try:
+        return int(value or 0)
+    except Exception:
+        return int(default)
+
+
+def _safe_float(value: object, default: float = 0.0) -> float:
+    try:
+        return float(value or 0.0)
+    except Exception:
+        return float(default)
+
+
+def _safe_text(value: object) -> str:
+    return str(value or "").strip()
+
+
+def _fetch_api(api_base: str, path: str) -> dict:
+    url = f"{api_base.rstrip('/')}/{path.lstrip('/')}"
+    try:
+        request = urllib.request.Request(url, headers={"User-Agent": "domaincheck-runtime-observer/0.1"})
+        with urllib.request.urlopen(request, timeout=8) as response:
+            payload = json.loads(response.read().decode("utf-8", errors="ignore"))
+        if isinstance(payload, dict):
+            data = payload.get("data")
+            if isinstance(data, dict):
+                return data
+            return payload
+        return {"_error": f"unexpected payload type: {type(payload).__name__}", "_url": url}
+    except urllib.error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="ignore")
+        return {"_error": f"http {exc.code}", "_body": body[:1000], "_url": url}
+    except Exception as exc:
+        return {"_error": str(exc), "_url": url}
+
+
+def _observe_payload(api_base: str) -> dict:
+    runtime_status = _fetch_api(api_base, "runtime/status")
+    direct_active_job = _fetch_api(api_base, "detect/job/active")
+    runtime_active_job = dict(((runtime_status.get("detect") or {}).get("active_job") or {}))
+    readiness = _fetch_api(api_base, "runtime/readiness")
+    sync_summary = _fetch_api(api_base, "runtime/sync-summary")
+    active_job = _merge_active_job_payloads(direct_active_job, runtime_active_job)
+    return {
+        "runtime_status": runtime_status,
+        "active_job": active_job,
+        "direct_active_job": direct_active_job,
+        "runtime_active_job": runtime_active_job,
+        "readiness": readiness,
+        "sync_summary": sync_summary,
+    }
+
+
+def _real_node_count(active_job: dict) -> int:
+    count = 0
+    for item in list((active_job or {}).get("node_stats") or []):
+        node_code = _safe_text((item or {}).get("node_code"))
+        if node_code and node_code != "unassigned":
+            count += 1
+    return count
+
+
+def _merge_active_job_payloads(direct_active_job: dict, runtime_active_job: dict) -> dict:
+    direct_active_job = dict(direct_active_job or {})
+    runtime_active_job = dict(runtime_active_job or {})
+    if not runtime_active_job:
+        return direct_active_job
+    if not direct_active_job:
+        return runtime_active_job
+
+    merged = dict(direct_active_job)
+    same_job = (
+        _safe_int(direct_active_job.get("job_id")) > 0
+        and _safe_int(direct_active_job.get("job_id")) == _safe_int(runtime_active_job.get("job_id"))
+    ) or (
+        _safe_text(direct_active_job.get("job_code"))
+        and _safe_text(direct_active_job.get("job_code")) == _safe_text(runtime_active_job.get("job_code"))
+    )
+    if not same_job:
+        return direct_active_job
+
+    preferred_runtime_keys = {
+        "node_stats",
+        "distributed_node_stats",
+        "display_items_claimed",
+        "display_items_running",
+        "display_current_load",
+        "display_active_threads",
+        "display_max_threads",
+        "display_items_completed",
+        "display_items_failed",
+        "display_active_node_codes",
+        "processed_recent",
+        "processed_per_minute",
+        "completed_recent",
+        "failed_recent",
+        "blacklisted_recent",
+        "runtime_snapshot_job_id",
+        "runtime_snapshot_job_code",
+        "runtime_snapshot_queue",
+    }
+    runtime_is_richer = _real_node_count(runtime_active_job) >= _real_node_count(direct_active_job)
+    for key in preferred_runtime_keys:
+        if key in runtime_active_job and runtime_is_richer:
+            merged[key] = runtime_active_job.get(key)
+    return merged
+
+
+def _active_node_rows(active_job: dict) -> list[dict]:
+    rows: list[dict] = []
+    for raw_item in list(active_job.get("node_stats") or []):
+        item = dict(raw_item or {})
+        node_code = _safe_text(item.get("node_code"))
+        if not node_code or node_code == "unassigned":
+            continue
+        current_load = max(
+            _safe_int(item.get("current_load")),
+            _safe_int(item.get("active_threads")),
+            _safe_int(item.get("display_running")),
+            _safe_int(item.get("items_running")),
+        )
+        processed_recent = _safe_int(item.get("processed_recent"))
+        items_claimed = _safe_int(item.get("items_claimed"))
+        status = _safe_text(item.get("status"))
+        if current_load <= 0 and processed_recent <= 0 and items_claimed <= 0 and status not in {"busy", "online"}:
+            continue
+        rows.append(
+            {
+                "node_code": node_code,
+                "status": status or "-",
+                "region": _safe_text(item.get("region")) or "-",
+                "role": _safe_text(item.get("role")) or "-",
+                "current_load": current_load,
+                "display_running": _safe_int(item.get("display_running")),
+                "active_threads": _safe_int(item.get("active_threads")),
+                "max_threads": _safe_int(item.get("max_threads")),
+                "items_claimed": items_claimed,
+                "items_running": _safe_int(item.get("items_running")),
+                "processed_recent": processed_recent,
+                "processed_per_minute": _safe_float(item.get("processed_per_minute")),
+                "completed_recent": _safe_int(item.get("completed_recent")),
+                "failed_recent": _safe_int(item.get("failed_recent")),
+                "blacklisted_recent": _safe_int(item.get("blacklisted_recent")),
+                "last_heartbeat_at": _safe_text(item.get("last_heartbeat_at")),
+            }
+        )
+    rows.sort(
+        key=lambda item: (
+            -int(item.get("current_load", 0)),
+            -int(item.get("processed_recent", 0)),
+            -int(item.get("items_claimed", 0)),
+            str(item.get("node_code") or ""),
+        )
+    )
+    return rows
+
+
+def _step_rows(active_job: dict) -> list[dict]:
+    rows: list[dict] = []
+    for raw_item in list(active_job.get("step_stats") or []):
+        item = dict(raw_item or {})
+        rows.append(
+            {
+                "step_code": _safe_text(item.get("step_code")),
+                "step_name": _safe_text(item.get("step_name")),
+                "pending": _safe_int(item.get("items_pending")),
+                "running": _safe_int(item.get("items_running")),
+                "completed": _safe_int(item.get("items_completed")),
+                "failed": _safe_int(item.get("items_failed")),
+                "blacklisted": _safe_int(item.get("items_blacklisted")),
+            }
+        )
+    rows.sort(key=lambda item: (-int(item["pending"]), -int(item["running"]), item["step_name"]))
+    return rows
+
+
+def _summary_numbers(payload: dict) -> dict:
+    runtime_status = dict(payload.get("runtime_status") or {})
+    active_job = dict(payload.get("active_job") or {})
+    active_nodes = _active_node_rows(active_job)
+    active_processes = len(active_nodes)
+    active_threads = max(
+        _safe_int(active_job.get("display_active_threads")),
+        _safe_int(active_job.get("display_items_running")),
+        sum(max(_safe_int(item.get("current_load")), _safe_int(item.get("display_running"))) for item in active_nodes),
+    )
+    max_threads = max(
+        _safe_int(active_job.get("display_max_threads")),
+        sum(_safe_int(item.get("max_threads")) for item in active_nodes if _safe_int(item.get("max_threads")) > 0),
+    )
+    processed_recent = _safe_int(active_job.get("processed_recent"))
+    failed_recent = _safe_int(active_job.get("failed_recent"))
+    blacklisted_recent = _safe_int(active_job.get("blacklisted_recent"))
+    items_pending = _safe_int(active_job.get("items_pending"))
+    items_claimed = _safe_int(active_job.get("items_claimed"))
+    items_running = _safe_int(active_job.get("items_running"))
+    items_completed = _safe_int(active_job.get("items_completed"))
+    items_failed = _safe_int(active_job.get("items_failed"))
+    items_blacklisted = _safe_int(active_job.get("items_blacklisted"))
+    return {
+        "job_id": _safe_int(active_job.get("job_id")),
+        "job_code": _safe_text(active_job.get("job_code")),
+        "job_status": _safe_text(active_job.get("status")) or "-",
+        "progress_percent": _safe_float(active_job.get("progress_percent")),
+        "active_processes": active_processes,
+        "active_threads": active_threads,
+        "max_threads": max_threads,
+        "processed_recent": processed_recent,
+        "processed_per_minute": _safe_float(active_job.get("processed_per_minute")),
+        "completed_recent": _safe_int(active_job.get("completed_recent")),
+        "failed_recent": failed_recent,
+        "blacklisted_recent": blacklisted_recent,
+        "items_pending": items_pending,
+        "items_claimed": items_claimed,
+        "items_running": items_running,
+        "items_completed": items_completed,
+        "items_failed": items_failed,
+        "items_blacklisted": items_blacklisted,
+        "online_worker_nodes": _safe_int(((runtime_status.get("cluster_summary") or {}).get("online_worker_nodes"))),
+        "online_control_nodes": _safe_int(((runtime_status.get("cluster_summary") or {}).get("online_control_nodes"))),
+    }
+
+
+def _diagnose_state(summary: dict) -> tuple[str, str]:
+    items_pending = int(summary["items_pending"])
+    active_processes = int(summary["active_processes"])
+    active_threads = int(summary["active_threads"])
+    processed_recent = int(summary["processed_recent"])
+    items_running = int(summary["items_running"])
+
+    if active_processes > 0 and active_threads > 0 and processed_recent > 0:
+        return "真跑中", "已经有真实执行面和最近吞吐，不是只剩显示残影。"
+    if active_processes > 0 and active_threads > 0 and items_running > 0:
+        return "在跑但偏慢", "有执行面，也有运行中任务，但最近吞吐还没完全拉起来。"
+    if items_pending > 0 and active_processes == 0 and active_threads == 0:
+        return "没跑起来", "队列还有积压，但当前没看到真实执行面在消化。"
+    if items_pending > 0 and processed_recent == 0:
+        return "疑似残影", "队列还有积压，但最近没有看到处理增量，需要继续查执行链。"
+    return "观察中", "当前口径没有明确异常，但还需要继续看最近吞吐是否持续变化。"
+
+
+def _line(char: str = "-") -> str:
+    width = max(60, min(120, shutil.get_terminal_size((100, 20)).columns))
+    return char * width
+
+
+def _render_summary(payload: dict, *, top_nodes: int) -> str:
+    runtime_status = dict(payload.get("runtime_status") or {})
+    active_job = dict(payload.get("active_job") or {})
+    readiness = dict(payload.get("readiness") or {})
+    sync_summary = dict(payload.get("sync_summary") or {})
+    summary = _summary_numbers(payload)
+    state_label, state_reason = _diagnose_state(summary)
+    active_nodes = _active_node_rows(active_job)
+    step_rows = _step_rows(active_job)
+    lines: list[str] = []
+
+    lines.append(_line("="))
+    lines.append(f"domainCheck 运行观察面板  {_now_text()}")
+    lines.append(_line("="))
+    lines.append(f"状态判断: {state_label}")
+    lines.append(f"判断理由: {state_reason}")
+    lines.append(
+        f"当前任务: job_id={summary['job_id']}  job_code={summary['job_code'] or '-'}  status={summary['job_status']}  进度={summary['progress_percent']:.2f}%"
+    )
+    lines.append(
+        f"就绪状态: { _safe_text(readiness.get('status')) or '-' }  摘要: { _safe_text(readiness.get('summary')) or '-' }"
+    )
+    lines.append(
+        f"集群在线: mainland worker={summary['online_worker_nodes']}  mainland controller={summary['online_control_nodes']}"
+    )
+
+    lines.append(_line())
+    lines.append("一眼先看这 4 组数:")
+    lines.append(
+        f"任务积压: pending={summary['items_pending']}  claimed={summary['items_claimed']}  running={summary['items_running']}"
+    )
+    lines.append(
+        f"结果产出: completed={summary['items_completed']}  failed={summary['items_failed']}  blacklisted={summary['items_blacklisted']}"
+    )
+    lines.append(
+        f"执行面: active_processes={summary['active_processes']}  active_threads={summary['active_threads']}  max_threads={summary['max_threads']}"
+    )
+    lines.append(
+        f"最近吞吐: processed_recent={summary['processed_recent']}  per_minute={summary['processed_per_minute']:.2f}  failed_recent={summary['failed_recent']}  blacklisted_recent={summary['blacklisted_recent']}"
+    )
+
+    detect_runtime = dict(((runtime_status.get("detect") or {}).get("active_job") or {}))
+    if detect_runtime:
+        lines.append(_line())
+        lines.append(
+            "运行口径提示: "
+            f"display_running={_safe_int(detect_runtime.get('display_items_running'))}  "
+            f"display_threads={_safe_int(detect_runtime.get('display_active_threads'))}  "
+            f"display_max={_safe_int(detect_runtime.get('display_max_threads'))}"
+        )
+
+    lines.append(_line())
+    lines.append("当前真正有动作的节点:")
+    if not active_nodes:
+        lines.append("- 暂时没看到有真实负载或最近吞吐的节点。")
+    else:
+        for item in active_nodes[: max(1, int(top_nodes or 8))]:
+            lines.append(
+                "- "
+                f"{item['node_code']}  "
+                f"load={item['current_load']}  "
+                f"running={item['display_running']}  "
+                f"max={item['max_threads']}  "
+                f"processed_recent={item['processed_recent']}  "
+                f"failed_recent={item['failed_recent']}  "
+                f"blacklisted_recent={item['blacklisted_recent']}  "
+                f"status={item['status']}"
+            )
+
+    lines.append(_line())
+    lines.append("步骤分布:")
+    for item in step_rows:
+        lines.append(
+            "- "
+            f"{item['step_name'] or item['step_code']}  "
+            f"pending={item['pending']}  running={item['running']}  "
+            f"completed={item['completed']}  failed={item['failed']}  blacklisted={item['blacklisted']}"
+        )
+
+    detect_batches = dict((sync_summary.get("detect_result_batches") or {}))
+    if sync_summary:
+        lines.append(_line())
+        lines.append(
+            "结果回传: "
+            f"enabled={_safe_text(sync_summary.get('enabled')) or '-'}  "
+            f"pending_batches={_safe_int(detect_batches.get('pending'))}  "
+            f"pushing_batches={_safe_int(detect_batches.get('pushing'))}  "
+            f"failed_batches={_safe_int(detect_batches.get('failed'))}  "
+            f"synced_batches={_safe_int(detect_batches.get('synced'))}"
+        )
+
+    warnings = list(readiness.get("warnings") or [])
+    info = list(readiness.get("info") or [])
+    if warnings or info:
+        lines.append(_line())
+        lines.append("当前提示:")
+        for text in warnings[:5]:
+            lines.append(f"- warning: {_safe_text(text)}")
+        for text in info[:3]:
+            lines.append(f"- info: {_safe_text(text)}")
+
+    lines.append(_line("="))
+    lines.append("建议:")
+    lines.append("- 先盯 `任务积压 / 执行面 / 最近吞吐`，这三组一起动，才算真跑。")
+    lines.append("- `日志` 只拿来辅助定位，不要拿日志多少判断是不是在跑。")
+    lines.append("- 如果 `pending` 很高，但 `active_threads` 和 `processed_recent` 都接近 0，就是没跑起来。")
+    lines.append("- 如果 `failed_recent` 很高而 `completed_recent`、`blacklisted_recent` 很低，说明更像外部步骤超时，不是黑名单在推进。")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Render a human-friendly runtime observation panel.")
+    parser.add_argument("--api-base", default=os.getenv("DOMAINCHECK_API_BASE", DEFAULT_API_BASE))
+    parser.add_argument("--watch", type=int, default=0, help="refresh interval in seconds; 0 means run once")
+    parser.add_argument("--top-nodes", type=int, default=10)
+    parser.add_argument("--json", action="store_true", help="print raw normalized payload instead of panel")
+    args = parser.parse_args()
+
+    while True:
+        payload = _observe_payload(args.api_base)
+        if args.json:
+            print(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
+        else:
+            if int(args.watch or 0) > 0:
+                print("\033[2J\033[H", end="")
+            print(_render_summary(payload, top_nodes=max(1, int(args.top_nodes or 10))))
+        if int(args.watch or 0) <= 0:
+            return 0
+        time.sleep(max(2, int(args.watch)))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/unstick_rebuild_release_index.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Repeatedly kill blockers holding DROP INDEX CONCURRENTLY for release index."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import psycopg2
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_release_node_job"
+POLL_SECONDS = 3
+# Large enough to keep unblocking for roughly an hour without manual babysitting.
+MAX_LOOPS = 1200
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    conn = psycopg2.connect(
+        host="127.0.0.1",
+        port=5432,
+        user=os.getenv("DB_USER", "postgres"),
+        password=os.getenv("DB_PASSWORD"),
+        dbname="domain",
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        for loop in range(MAX_LOOPS):
+            cur.execute(
+                """
+                SELECT pid
+                FROM pg_stat_activity
+                WHERE pid <> pg_backend_pid()
+                  AND state <> 'idle'
+                  AND query ILIKE %s
+                ORDER BY pid
+                """,
+                (f"%DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",),
+            )
+            drop_pids = [r[0] for r in cur.fetchall()]
+            if not drop_pids:
+                log("no_drop_pid")
+                break
+
+            blockers: list[int] = []
+            for pid in drop_pids:
+                cur.execute("SELECT pg_blocking_pids(%s)", (pid,))
+                row = cur.fetchone()
+                if row and row[0]:
+                    blockers.extend(int(v) for v in row[0])
+
+            blockers = sorted(set(blockers))
+            log(f"loop={loop} drop_pids={drop_pids} blockers={blockers}")
+            if not blockers:
+                time.sleep(POLL_SECONDS)
+                continue
+
+            for pid in blockers:
+                cur.execute("SELECT pg_cancel_backend(%s), pg_terminate_backend(%s)", (pid, pid))
+                log(f"kill pid={pid} result={cur.fetchone()}")
+            time.sleep(POLL_SECONDS)
+    finally:
+        cur.close()
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/unstick_rebuild_stalled_index.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Repeatedly kill blockers holding DROP INDEX CONCURRENTLY for stalled index."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import psycopg2
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_stalled_job_activity"
+POLL_SECONDS = 3
+MAX_LOOPS = 1200
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    conn = psycopg2.connect(
+        host="127.0.0.1",
+        port=5432,
+        user=os.getenv("DB_USER", "postgres"),
+        password=os.getenv("DB_PASSWORD"),
+        dbname="domain",
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        for loop in range(MAX_LOOPS):
+            cur.execute(
+                """
+                SELECT pid
+                FROM pg_stat_activity
+                WHERE pid <> pg_backend_pid()
+                  AND state <> 'idle'
+                  AND query ILIKE %s
+                ORDER BY pid
+                """,
+                (f"%DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",),
+            )
+            drop_pids = [r[0] for r in cur.fetchall()]
+            if not drop_pids:
+                log("no_drop_pid")
+                break
+
+            blockers: list[int] = []
+            for pid in drop_pids:
+                cur.execute("SELECT pg_blocking_pids(%s)", (pid,))
+                row = cur.fetchone()
+                if row and row[0]:
+                    blockers.extend(int(v) for v in row[0])
+
+            blockers = sorted(set(blockers))
+            log(f"loop={loop} drop_pids={drop_pids} blockers={blockers}")
+            if not blockers:
+                time.sleep(POLL_SECONDS)
+                continue
+
+            for pid in blockers:
+                cur.execute("SELECT pg_cancel_backend(%s), pg_terminate_backend(%s)", (pid, pid))
+                log(f"kill pid={pid} result={cur.fetchone()}")
+            time.sleep(POLL_SECONDS)
+    finally:
+        cur.close()
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/watch_job876_tail_release.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Watch the release index build and reclaim known stuck job-876 tails."""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DOMAINCHECK_ROOT = ROOT / "domainCheck"
+if str(DOMAINCHECK_ROOT) not in sys.path:
+    sys.path.insert(0, str(DOMAINCHECK_ROOT))
+
+from app.utils.database import Database  # noqa: E402
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_release_node_job"
+JOB_ID = 876
+TARGET_NODES = [
+    "mainland-controller-01-ba",
+    "mainland-controller-01-bd",
+    "mainland-controller-01-aw",
+    "mainland-controller-01-ae",
+]
+POLL_SECONDS = 15
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def fetch_index_state(db: Database) -> tuple[bool, bool, str | None, int | None, int | None]:
+    conn = db.get_connection()
+    try:
+        cur = conn.cursor()
+        try:
+            cur.execute(
+                """
+                SELECT
+                    i.indisvalid,
+                    i.indisready,
+                    p.phase,
+                    p.blocks_done,
+                    p.blocks_total
+                FROM pg_index i
+                JOIN pg_class c ON c.oid = i.indexrelid
+                LEFT JOIN pg_stat_progress_create_index p
+                    ON p.index_relid = i.indexrelid
+                WHERE c.relname = %s
+                """,
+                (INDEX_NAME,),
+            )
+            row = cur.fetchone()
+        finally:
+            cur.close()
+    finally:
+        db.close(conn=conn)
+    if not row:
+        return False, False, None, None, None
+    return bool(row[0]), bool(row[1]), row[2], row[3], row[4]
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    db = Database()
+    log(f"watch start index={INDEX_NAME} job_id={JOB_ID} targets={','.join(TARGET_NODES)}")
+    while True:
+        try:
+            valid, ready, phase, blocks_done, blocks_total = fetch_index_state(db)
+            log(
+                "index_state "
+                f"valid={int(valid)} ready={int(ready)} "
+                f"phase={phase or 'none'} blocks_done={blocks_done} blocks_total={blocks_total}"
+            )
+            if valid:
+                break
+        except Exception as exc:  # pragma: no cover - operational script
+            log(f"index_state_error error={exc!r}")
+            db = Database()
+        time.sleep(POLL_SECONDS)
+
+    released_total = 0
+    for node in TARGET_NODES:
+        try:
+            released = db.release_detect_job_items_for_node_job(node, JOB_ID)
+            released_total += int(released or 0)
+            log(f"release node={node} released={released}")
+        except Exception as exc:  # pragma: no cover - operational script
+            log(f"release_error node={node} error={exc!r}")
+
+    try:
+        status = db.refresh_detect_job_status(JOB_ID)
+        log(f"refresh_status job_id={JOB_ID} status={status}")
+    except Exception as exc:  # pragma: no cover - operational script
+        log(f"refresh_status_error job_id={JOB_ID} error={exc!r}")
+
+    log(f"done released_total={released_total}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())