feat: stabilize multi-region runtime sync and worker orchestration
This commit is contained in:
@@ -11,6 +11,23 @@ from app.services.ops_agent_service import ensure_ops_agent_schema
|
||||
from app.services.ops_job_service import ensure_ops_schema
|
||||
from app.services.ops_release_service import ensure_ops_release_schema
|
||||
_heartbeat_stop_event = threading.Event()
|
||||
_bootstrap_started = False
|
||||
_bootstrap_lock = threading.Lock()
|
||||
|
||||
|
||||
def _bootstrap_runtime_prerequisites() -> None:
|
||||
for task in (
|
||||
ensure_runtime_schema,
|
||||
ensure_ops_schema,
|
||||
ensure_ops_agent_schema,
|
||||
ensure_ops_release_schema,
|
||||
register_local_control_heartbeat,
|
||||
):
|
||||
try:
|
||||
task()
|
||||
except Exception:
|
||||
# Startup must not block API listening on slow schema checks or DB stalls.
|
||||
pass
|
||||
|
||||
|
||||
def _control_heartbeat_loop() -> None:
|
||||
@@ -39,14 +56,18 @@ app.add_middleware(
|
||||
|
||||
@app.on_event("startup")
|
||||
def on_startup() -> None:
|
||||
ensure_runtime_schema()
|
||||
ensure_ops_schema()
|
||||
ensure_ops_agent_schema()
|
||||
ensure_ops_release_schema()
|
||||
global _bootstrap_started
|
||||
remember_registered_route_paths(route.path for route in app.routes)
|
||||
register_local_control_heartbeat()
|
||||
_heartbeat_stop_event.clear()
|
||||
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
|
||||
with _bootstrap_lock:
|
||||
if not _bootstrap_started:
|
||||
threading.Thread(
|
||||
target=_bootstrap_runtime_prerequisites,
|
||||
name="api-bootstrap",
|
||||
daemon=True,
|
||||
).start()
|
||||
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
|
||||
_bootstrap_started = True
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
|
||||
Reference in New Issue
Block a user