feat: stabilize multi-region runtime sync and worker orchestration

This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions

View File

@@ -11,6 +11,23 @@ from app.services.ops_agent_service import ensure_ops_agent_schema
from app.services.ops_job_service import ensure_ops_schema
from app.services.ops_release_service import ensure_ops_release_schema
_heartbeat_stop_event = threading.Event()
_bootstrap_started = False
_bootstrap_lock = threading.Lock()
def _bootstrap_runtime_prerequisites() -> None:
for task in (
ensure_runtime_schema,
ensure_ops_schema,
ensure_ops_agent_schema,
ensure_ops_release_schema,
register_local_control_heartbeat,
):
try:
task()
except Exception:
# Startup must not block API listening on slow schema checks or DB stalls.
pass
def _control_heartbeat_loop() -> None:
@@ -39,14 +56,18 @@ app.add_middleware(
@app.on_event("startup")
def on_startup() -> None:
ensure_runtime_schema()
ensure_ops_schema()
ensure_ops_agent_schema()
ensure_ops_release_schema()
global _bootstrap_started
remember_registered_route_paths(route.path for route in app.routes)
register_local_control_heartbeat()
_heartbeat_stop_event.clear()
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
with _bootstrap_lock:
if not _bootstrap_started:
threading.Thread(
target=_bootstrap_runtime_prerequisites,
name="api-bootstrap",
daemon=True,
).start()
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
_bootstrap_started = True
@app.on_event("shutdown")