feat: stabilize multi-region runtime sync and worker orchestration

2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
--- a/tools/rebuild_stalled_job_activity_index.py
+++ b/tools/rebuild_stalled_job_activity_index.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Drop and recreate idx_detect_job_items_stalled_job_activity on mainland."""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+
+import psycopg2
+from psycopg2 import sql
+
+
+ENV_FILE = Path("/etc/default/domaincheck-worker")
+INDEX_NAME = "idx_detect_job_items_stalled_job_activity"
+
+
+def load_env(path: Path) -> None:
+    if not path.exists():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        os.environ[key] = value
+
+
+def log(message: str) -> None:
+    stamp = time.strftime("%Y-%m-%d %H:%M:%S")
+    print(f"[{stamp}] {message}", flush=True)
+
+
+def fetch_index_state(cur: psycopg2.extensions.cursor, index_name: str) -> tuple[bool, bool, bool] | None:
+    cur.execute(
+        """
+        SELECT i.indisvalid, i.indisready, i.indislive
+        FROM pg_index i
+        JOIN pg_class c ON c.oid = i.indexrelid
+        WHERE c.relname = %s
+        """,
+        (index_name,),
+    )
+    row = cur.fetchone()
+    if not row:
+        return None
+    return bool(row[0]), bool(row[1]), bool(row[2])
+
+
+def fetch_conflicting_backends(cur: psycopg2.extensions.cursor) -> list[tuple[int, str, str]]:
+    cur.execute(
+        """
+        SELECT pid, query, coalesce(wait_event_type, '')
+        FROM pg_stat_activity
+        WHERE state <> 'idle'
+          AND (
+            query ILIKE %s OR
+            query ILIKE %s
+          )
+        ORDER BY backend_start
+        """,
+        (
+            "%idx_detect_job_items_stalled_job_activity%",
+            "%idx_detect_job_items_release_node_job%",
+        ),
+    )
+    rows = []
+    for pid, query, wait_event_type in cur.fetchall():
+        snippet = " ".join((query or "").split())[:160]
+        rows.append((pid, wait_event_type, snippet))
+    return rows
+
+
+def main() -> int:
+    load_env(ENV_FILE)
+    conn = psycopg2.connect(
+        host="127.0.0.1",
+        port=5432,
+        user=os.getenv("DB_USER", "postgres"),
+        password=os.getenv("DB_PASSWORD"),
+        dbname="domain",
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+    try:
+        conflicts = fetch_conflicting_backends(cur)
+        if conflicts:
+            log(f"preflight conflicting_backends={conflicts}")
+
+        log("drop start")
+        cur.execute(sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(sql.Identifier(INDEX_NAME)))
+        log("drop done")
+
+        residual_state = fetch_index_state(cur, INDEX_NAME)
+        if residual_state is not None:
+            raise RuntimeError(
+                f"index shell still present after drop: name={INDEX_NAME} "
+                f"state={residual_state}"
+            )
+
+        log("create start")
+        cur.execute(
+            sql.SQL(
+                """
+                CREATE INDEX CONCURRENTLY {}
+                ON public.detect_job_items (
+                    job_id,
+                    status,
+                    (COALESCE(updated_at, started_at, create_time)),
+                    id
+                )
+                WHERE status IN ('claimed', 'running')
+                """
+            ).format(sql.Identifier(INDEX_NAME))
+        )
+        log("create done")
+        log(f"final state={fetch_index_state(cur, INDEX_NAME)}")
+    finally:
+        cur.close()
+        conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())