admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
domain-api/app/api/routes/detect.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
from uuid import uuid4
from fastapi import APIRouter
from fastapi import APIRouter, Body
from app.core.config import settings
from app.schemas.common import ApiResponse
@@ -18,7 +18,7 @@ from app.services.detect_job_service import (
from app.services.detect_service import get_detect_status
from app.services.detect_run_service import create_detect_run_snapshot, finalize_detect_run, mark_detect_run_stopping
from app.services.ops_job_service import create_ops_job, list_managed_nodes
from app.services.settings_service import get_settings_payload, resolve_thread_count
from app.services.settings_service import get_settings_payload, resolve_process_count, resolve_thread_count
from app.services.worker_control_service import send_worker_command, start_worker
router = APIRouter(tags=["detect"])
@@ -66,7 +66,13 @@ def _build_detect_action_result(
def _build_settings_summary(settings_payload: dict) -> dict:
thread_count_resolution = resolve_thread_count(settings_payload=settings_payload)
process_count_resolution = resolve_process_count(settings_payload=settings_payload)
return {
"process_count": int(process_count_resolution["effective_process_count"]),
"process_count_default": int(process_count_resolution["default_process_count"]),
"process_count_source": str(process_count_resolution["source"]),
"process_count_override": process_count_resolution["override_process_count"],
"process_count_node_code": str(process_count_resolution["node_code"]),
"thread_count": int(thread_count_resolution["effective_thread_count"]),
"thread_count_default": int(thread_count_resolution["default_thread_count"]),
"thread_count_source": str(thread_count_resolution["source"]),
@@ -78,6 +84,38 @@ def _build_settings_summary(settings_payload: dict) -> dict:
}
def _normalize_target_node_codes(payload: dict | None) -> list[str]:
if not isinstance(payload, dict):
return []
normalized_targets: list[str] = []
def append_target(raw_value: object) -> None:
normalized_value = str(raw_value or "").strip()
if normalized_value and normalized_value not in normalized_targets:
normalized_targets.append(normalized_value)
for key in ("target_node_codes", "node_codes"):
raw_value = payload.get(key)
if isinstance(raw_value, (list, tuple, set)):
for item in raw_value:
append_target(item)
elif isinstance(raw_value, str) and raw_value.strip():
for item in raw_value.split(","):
append_target(item)
if normalized_targets:
return normalized_targets
for key in ("target_node_code", "node_code"):
raw_value = payload.get(key)
if raw_value not in (None, ""):
append_target(raw_value)
if normalized_targets:
return normalized_targets
return normalized_targets
def _mainland_detect_targets() -> dict[str, list[dict]]:
controllers: list[dict] = []
workers: list[dict] = []
@@ -212,15 +250,18 @@ def _dispatch_remote_detect_start(*, job_summary: dict, cycle_token: str) -> dic
}
def _dispatch_remote_detect_stop(*, active_job: dict | None, cycle_token: str = "") -> dict:
def _dispatch_remote_detect_stop(*, active_job: dict | None, cycle_token: str = "", payload: dict | None = None) -> dict:
targets = _mainland_detect_targets()
job_summary = active_job or {}
queued: list[dict] = []
target_node_codes = _normalize_target_node_codes(payload)
for node in [*targets["controllers"], *targets["workers"]]:
node_code = str(node.get("node_code") or "").strip()
if not node_code:
continue
if target_node_codes and node_code not in target_node_codes:
continue
queued.append(
_queue_remote_detect_job(
node_code=node_code,
@@ -317,26 +358,63 @@ def start_detect(step_code: str | None = None) -> ApiResponse:
local_worker_expected = _local_worker_expected_on_this_node()
if local_worker_expected:
ok, message = start_worker()
if not ok:
result = _build_detect_action_result(
action="start",
ok=False,
message=message,
data={"job": job_summary},
)
prestart_snapshot = get_detect_status()
worker_already_running = bool(prestart_snapshot.get("worker_online", False)) or int(
prestart_snapshot.get("worker_process_count", 0) or 0
) > 0
if worker_already_running:
ok = True
message = "检测端已在运行，跳过重复启动，直接发送控制指令"
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_failed",
level="error",
message=f"启动 Worker 失败: {message}",
payload={"cycle_token": cycle_token},
)
return ApiResponse(
code=1,
event_type="job_dispatch_start_skipped",
level="info",
message=message,
data=result,
payload={
"cycle_token": cycle_token,
"worker_process_count": int(prestart_snapshot.get("worker_process_count", 0) or 0),
},
)
else:
ok, message = start_worker()
if not ok:
degraded_snapshot = get_detect_status()
worker_already_running = bool(degraded_snapshot.get("worker_online", False)) or int(
degraded_snapshot.get("worker_process_count", 0) or 0
) > 0
if not worker_already_running:
result = _build_detect_action_result(
action="start",
ok=False,
message=message,
data={"job": job_summary},
)
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_failed",
level="error",
message=f"启动 Worker 失败: {message}",
payload={"cycle_token": cycle_token},
)
return ApiResponse(
code=1,
message=message,
data=result,
)
degraded_message = f"{message}；检测端已在运行，改为直接发送控制指令"
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_start_degraded",
level="warning",
message=degraded_message,
payload={
"cycle_token": cycle_token,
"worker_process_count": int(degraded_snapshot.get("worker_process_count", 0) or 0),
},
)
ok = True
message = degraded_message
command_ok, command_message = send_worker_command(
"start_detection",
@@ -372,18 +450,19 @@ def start_detect(step_code: str | None = None) -> ApiResponse:
settings_summary = _build_settings_summary(settings_payload)
if command_ok:
remote_dispatch = _dispatch_remote_detect_start(job_summary=job_summary, cycle_token=cycle_token)
create_detect_run_snapshot(
message=f"{message}；{command_message}",
runtime={
"mode": snapshot.get("worker_mode", ""),
"running": snapshot.get("worker_online", False),
"process_count": snapshot.get("worker_process_count", 0),
"latest_start_time": snapshot.get("worker_latest_start_time", ""),
"message": snapshot.get("worker_runtime_message", ""),
},
progress=snapshot.get("progress", {}),
settings_summary=settings_summary,
)
if local_worker_expected:
create_detect_run_snapshot(
message=f"{message}；{command_message}",
runtime={
"mode": snapshot.get("worker_mode", ""),
"running": snapshot.get("worker_online", False),
"process_count": snapshot.get("worker_process_count", 0),
"latest_start_time": snapshot.get("worker_latest_start_time", ""),
"message": snapshot.get("worker_runtime_message", ""),
},
progress=snapshot.get("progress", {}),
settings_summary=settings_summary,
)
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_remote_queued",
@@ -411,11 +490,16 @@ def start_detect(step_code: str | None = None) -> ApiResponse:
@router.post("/detect/stop", response_model=ApiResponse)
def stop_detect() -> ApiResponse:
def stop_detect(payload: dict | None = Body(default=None)) -> ApiResponse:
active_job = get_active_detect_job_summary(event_limit=10)
ok, message = send_worker_command("stop_detection")
normalized_payload = {
key: value
for key, value in dict(payload or {}).items()
if value not in (None, "")
}
ok, message = send_worker_command("stop_detection", payload=normalized_payload)
cycle_token = str((active_job or {}).get("current_cycle_token") or "").strip()
remote_dispatch = _dispatch_remote_detect_stop(active_job=active_job, cycle_token=cycle_token)
remote_dispatch = _dispatch_remote_detect_stop(active_job=active_job, cycle_token=cycle_token, payload=normalized_payload)
if active_job:
append_detect_job_event(
active_job["job_id"],

22
domain-api/app/api/routes/ops.py
View File

@@ -31,6 +31,11 @@ from app.services.ops_job_service import (
sync_managed_nodes_from_cluster,
upsert_managed_node,
)
from app.services.ops_migration_service import (
execute_ops_migration,
get_ops_migration_source_profile,
preview_ops_migration,
)
from app.services.ops_playbook_service import (
cancel_ops_playbook_run,
execute_ops_playbook,
@@ -80,6 +85,23 @@ def ops_overview() -> ApiResponse:
return ApiResponse(data=get_ops_overview())
@router.get("/ops/migration/source-profile", response_model=ApiResponse)
def ops_migration_source_profile() -> ApiResponse:
return ApiResponse(data=get_ops_migration_source_profile())
@router.post("/ops/migration/preview", response_model=ApiResponse)
def ops_migration_preview(payload: dict | None = None) -> ApiResponse:
ok, message, data = preview_ops_migration(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/migration/execute", response_model=ApiResponse)
def ops_migration_execute(payload: dict | None = None) -> ApiResponse:
ok, message, data = execute_ops_migration(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/link-snapshot", response_model=ApiResponse)
def ops_link_snapshot() -> ApiResponse:
return ApiResponse(data=get_ops_link_snapshot())

58
domain-api/app/api/routes/runtime.py
View File

@@ -19,6 +19,55 @@ from app.services.sync_record_service import get_sync_summary, list_sync_records
router = APIRouter(tags=["runtime"])
def _filter_debug_handover_by_node_code(payload: dict, node_code: str) -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return dict(payload or {})
normalized_payload = dict(payload or {})
def _matches_node(item: object) -> bool:
if not isinstance(item, dict):
return False
return str(item.get("node_code") or "").strip() == normalized_node_code
overview = dict(normalized_payload.get("overview") or {})
if overview:
overview["recent_issues"] = [
item
for item in list(overview.get("recent_issues") or [])
if _matches_node(item)
]
normalized_payload["overview"] = overview
normalized_payload["recent_issues"] = [
item
for item in list(normalized_payload.get("recent_issues") or [])
if _matches_node(item)
]
normalized_payload["issue_groups"] = [
item
for item in list(normalized_payload.get("issue_groups") or [])
if _matches_node(item)
]
failure_handoff = dict(normalized_payload.get("failure_handoff") or {})
if failure_handoff:
failure_handoff["recent_issues"] = [
item
for item in list(failure_handoff.get("recent_issues") or [])
if _matches_node(item)
]
failure_handoff["issue_groups"] = [
item
for item in list(failure_handoff.get("issue_groups") or [])
if _matches_node(item)
]
normalized_payload["failure_handoff"] = failure_handoff
return normalized_payload
@router.get("/runtime/status", response_model=ApiResponse)
def runtime_status() -> ApiResponse:
return ApiResponse(data=get_runtime_status())
@@ -60,6 +109,7 @@ def runtime_debug_events(
service: Optional[str] = None,
event_type: Optional[str] = None,
source_region: Optional[str] = None,
node_code: Optional[str] = None,
level: Optional[str] = None,
before_id: Optional[int] = None,
after_id: Optional[int] = None,
@@ -71,6 +121,7 @@ def runtime_debug_events(
service=service,
event_type=event_type,
source_region=source_region,
node_code=node_code,
level=level,
before_id=before_id,
after_id=after_id,
@@ -125,12 +176,7 @@ def runtime_health_handover(
)
normalized_node_code = str(node_code or "").strip()
if normalized_node_code:
data = dict(data)
data["nodes"] = [
item
for item in list(data.get("nodes") or [])
if str(item.get("node_code") or "").strip() == normalized_node_code
]
data = _filter_debug_handover_by_node_code(data, normalized_node_code)
return ApiResponse(data=data)