feat: stabilize multi-region runtime sync and worker orchestration

tools/collect_chinaz_gray_metrics.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+
+def run_ssh(remote_cmd: str, timeout: int) -> str:
+    proc = subprocess.run(
+        [
+            "python3",
+            "-c",
+            (
+                "import paramiko,sys;"
+                "host,user,password,cmd,timeout=sys.argv[1:6];"
+                "client=paramiko.SSHClient();"
+                "client.set_missing_host_key_policy(paramiko.AutoAddPolicy());"
+                "client.connect(host, username=user, password=password, timeout=int(timeout));"
+                "stdin,stdout,stderr=client.exec_command(cmd, timeout=int(timeout));"
+                "sys.stdout.write(stdout.read().decode('utf-8','ignore'));"
+                "err=stderr.read().decode('utf-8','ignore');"
+                "client.close();"
+                "sys.stderr.write(err)"
+            ),
+            os.environ["REMOTE_HOST"],
+            os.environ["REMOTE_USER"],
+            os.environ["REMOTE_PASS"],
+            remote_cmd,
+            str(timeout),
+        ],
+        capture_output=True,
+        text=True,
+        timeout=timeout + 10,
+    )
+    if proc.returncode != 0 and proc.stderr.strip():
+        raise RuntimeError(proc.stderr.strip())
+    return proc.stdout
+
+
+def parse_cpu_busy(text: str) -> Optional[float]:
+    match = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*id", text)
+    if not match:
+        return None
+    idle = float(match.group(1))
+    return round(max(0.0, 100.0 - idle), 2)
+
+
+def collect_snapshot(gray_workers: List[str], since: str) -> dict:
+    units = " ".join(f"-u domaincheck-worker@{worker}.service" for worker in gray_workers)
+    remote_cmd = (
+        "echo '###CPU'; "
+        "top -bn1 | grep '%Cpu' || true; "
+        "echo '###GRAY'; "
+        f"journalctl {units} --since '{since}' --no-pager "
+        "| grep -E 'proxy_direct_fallback|站长之家调用边界|single_step_finalized|重试预算耗尽' "
+        "| tail -n 1200 || true; "
+        "echo '###GLOBAL'; "
+        "journalctl -u 'domaincheck-worker@*.service' "
+        f"--since '{since}' --no-pager "
+        "| grep 'detect_order=detect_chinaz' | tail -n 1200 || true"
+    )
+    out = run_ssh(remote_cmd, timeout=40)
+    cpu_part = ""
+    gray_part = ""
+    global_part = ""
+    if "###GRAY" in out:
+        cpu_part, rest = out.split("###GRAY", 1)
+        if "###GLOBAL" in rest:
+            gray_part, global_part = rest.split("###GLOBAL", 1)
+        else:
+            gray_part = rest
+    else:
+        cpu_part = out
+    cpu_busy = parse_cpu_busy(cpu_part)
+
+    counts = {
+        "proxy_direct_fallback": 0,
+        "boundary_proxy_yes": 0,
+        "boundary_proxy_no": 0,
+        "single_step_finalized": 0,
+        "retry_budget_exhausted": 0,
+    }
+    per_worker: Dict[str, Dict[str, int]] = {}
+    for line in gray_part.splitlines():
+        worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
+        worker = worker_match.group(1) if worker_match else "-"
+        bucket = per_worker.setdefault(
+            worker,
+            {
+                "proxy_direct_fallback": 0,
+                "boundary_proxy_yes": 0,
+                "boundary_proxy_no": 0,
+                "single_step_finalized": 0,
+                "retry_budget_exhausted": 0,
+            },
+        )
+        if "proxy_direct_fallback" in line:
+            counts["proxy_direct_fallback"] += 1
+            bucket["proxy_direct_fallback"] += 1
+        if "站长之家调用边界" in line and "proxy=yes" in line:
+            counts["boundary_proxy_yes"] += 1
+            bucket["boundary_proxy_yes"] += 1
+        if "站长之家调用边界" in line and "proxy=no" in line:
+            counts["boundary_proxy_no"] += 1
+            bucket["boundary_proxy_no"] += 1
+        if "single_step_finalized" in line and "detect_key=detect_chinaz" in line:
+            counts["single_step_finalized"] += 1
+            bucket["single_step_finalized"] += 1
+        if "重试预算耗尽" in line and "站长之家检测" in line:
+            counts["retry_budget_exhausted"] += 1
+            bucket["retry_budget_exhausted"] += 1
+
+    global_worker_counts: Dict[str, int] = {}
+    if global_part.strip():
+        top_counts: Dict[str, int] = {}
+        for line in global_part.splitlines():
+            worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
+            if not worker_match:
+                continue
+            worker = worker_match.group(1)
+            top_counts[worker] = top_counts.get(worker, 0) + 1
+        global_worker_counts = dict(
+            sorted(top_counts.items(), key=lambda item: item[1], reverse=True)[:12]
+        )
+
+    return {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "cpu_busy_pct": cpu_busy,
+        "counts": counts,
+        "per_worker": per_worker,
+        "global_detect_chinaz_orders": sum(global_worker_counts.values()),
+        "global_detect_chinaz_top_workers": global_worker_counts,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--workers", required=True, help="comma-separated worker suffixes")
+    parser.add_argument("--duration-seconds", type=int, default=3600)
+    parser.add_argument("--interval-seconds", type=int, default=30)
+    parser.add_argument("--since-window-seconds", type=int, default=45)
+    args = parser.parse_args()
+
+    gray_workers = [item.strip() for item in args.workers.split(",") if item.strip()]
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+    deadline = time.time() + max(0, args.duration_seconds)
+    while time.time() <= deadline:
+        since_ts = datetime.fromtimestamp(
+            time.time() - max(1, args.since_window_seconds),
+            tz=timezone.utc,
+        ).astimezone().strftime("%Y-%m-%d %H:%M:%S")
+        snapshot = collect_snapshot(gray_workers, since_ts)
+        with open(args.output, "a", encoding="utf-8") as fh:
+            fh.write(json.dumps(snapshot, ensure_ascii=False) + "\n")
+        time.sleep(max(1, args.interval_seconds))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())