getDomain/tools/collect_chinaz_gray_metrics.py

#!/usr/bin/env python3
import argparse
import json
import os
import re
import subprocess
import time
from datetime import datetime, timezone
from typing import Dict, List, Optional


def run_ssh(remote_cmd: str, timeout: int) -> str:
    proc = subprocess.run(
        [
            "python3",
            "-c",
            (
                "import paramiko,sys;"
                "host,user,password,cmd,timeout=sys.argv[1:6];"
                "client=paramiko.SSHClient();"
                "client.set_missing_host_key_policy(paramiko.AutoAddPolicy());"
                "client.connect(host, username=user, password=password, timeout=int(timeout));"
                "stdin,stdout,stderr=client.exec_command(cmd, timeout=int(timeout));"
                "sys.stdout.write(stdout.read().decode('utf-8','ignore'));"
                "err=stderr.read().decode('utf-8','ignore');"
                "client.close();"
                "sys.stderr.write(err)"
            ),
            os.environ["REMOTE_HOST"],
            os.environ["REMOTE_USER"],
            os.environ["REMOTE_PASS"],
            remote_cmd,
            str(timeout),
        ],
        capture_output=True,
        text=True,
        timeout=timeout + 10,
    )
    if proc.returncode != 0 and proc.stderr.strip():
        raise RuntimeError(proc.stderr.strip())
    return proc.stdout


def parse_cpu_busy(text: str) -> Optional[float]:
    match = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*id", text)
    if not match:
        return None
    idle = float(match.group(1))
    return round(max(0.0, 100.0 - idle), 2)


def collect_snapshot(gray_workers: List[str], since: str) -> dict:
    units = " ".join(f"-u domaincheck-worker@{worker}.service" for worker in gray_workers)
    remote_cmd = (
        "echo '###CPU'; "
        "top -bn1 | grep '%Cpu' || true; "
        "echo '###GRAY'; "
        f"journalctl {units} --since '{since}' --no-pager "
        "| grep -E 'proxy_direct_fallback|站长之家调用边界|single_step_finalized|重试预算耗尽' "
        "| tail -n 1200 || true; "
        "echo '###GLOBAL'; "
        "journalctl -u 'domaincheck-worker@*.service' "
        f"--since '{since}' --no-pager "
        "| grep 'detect_order=detect_chinaz' | tail -n 1200 || true"
    )
    out = run_ssh(remote_cmd, timeout=40)
    cpu_part = ""
    gray_part = ""
    global_part = ""
    if "###GRAY" in out:
        cpu_part, rest = out.split("###GRAY", 1)
        if "###GLOBAL" in rest:
            gray_part, global_part = rest.split("###GLOBAL", 1)
        else:
            gray_part = rest
    else:
        cpu_part = out
    cpu_busy = parse_cpu_busy(cpu_part)

    counts = {
        "proxy_direct_fallback": 0,
        "boundary_proxy_yes": 0,
        "boundary_proxy_no": 0,
        "single_step_finalized": 0,
        "retry_budget_exhausted": 0,
    }
    per_worker: Dict[str, Dict[str, int]] = {}
    for line in gray_part.splitlines():
        worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
        worker = worker_match.group(1) if worker_match else "-"
        bucket = per_worker.setdefault(
            worker,
            {
                "proxy_direct_fallback": 0,
                "boundary_proxy_yes": 0,
                "boundary_proxy_no": 0,
                "single_step_finalized": 0,
                "retry_budget_exhausted": 0,
            },
        )
        if "proxy_direct_fallback" in line:
            counts["proxy_direct_fallback"] += 1
            bucket["proxy_direct_fallback"] += 1
        if "站长之家调用边界" in line and "proxy=yes" in line:
            counts["boundary_proxy_yes"] += 1
            bucket["boundary_proxy_yes"] += 1
        if "站长之家调用边界" in line and "proxy=no" in line:
            counts["boundary_proxy_no"] += 1
            bucket["boundary_proxy_no"] += 1
        if "single_step_finalized" in line and "detect_key=detect_chinaz" in line:
            counts["single_step_finalized"] += 1
            bucket["single_step_finalized"] += 1
        if "重试预算耗尽" in line and "站长之家检测" in line:
            counts["retry_budget_exhausted"] += 1
            bucket["retry_budget_exhausted"] += 1

    global_worker_counts: Dict[str, int] = {}
    if global_part.strip():
        top_counts: Dict[str, int] = {}
        for line in global_part.splitlines():
            worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line)
            if not worker_match:
                continue
            worker = worker_match.group(1)
            top_counts[worker] = top_counts.get(worker, 0) + 1
        global_worker_counts = dict(
            sorted(top_counts.items(), key=lambda item: item[1], reverse=True)[:12]
        )

    return {
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "cpu_busy_pct": cpu_busy,
        "counts": counts,
        "per_worker": per_worker,
        "global_detect_chinaz_orders": sum(global_worker_counts.values()),
        "global_detect_chinaz_top_workers": global_worker_counts,
    }


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--output", required=True)
    parser.add_argument("--workers", required=True, help="comma-separated worker suffixes")
    parser.add_argument("--duration-seconds", type=int, default=3600)
    parser.add_argument("--interval-seconds", type=int, default=30)
    parser.add_argument("--since-window-seconds", type=int, default=45)
    args = parser.parse_args()

    gray_workers = [item.strip() for item in args.workers.split(",") if item.strip()]
    os.makedirs(os.path.dirname(args.output), exist_ok=True)
    deadline = time.time() + max(0, args.duration_seconds)
    while time.time() <= deadline:
        since_ts = datetime.fromtimestamp(
            time.time() - max(1, args.since_window_seconds),
            tz=timezone.utc,
        ).astimezone().strftime("%Y-%m-%d %H:%M:%S")
        snapshot = collect_snapshot(gray_workers, since_ts)
        with open(args.output, "a", encoding="utf-8") as fh:
            fh.write(json.dumps(snapshot, ensure_ascii=False) + "\n")
        time.sleep(max(1, args.interval_seconds))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())