feat: add ops center and node onboarding flow

This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions

View File

@@ -0,0 +1,494 @@
from __future__ import annotations
from app.core.config import settings
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.ops_execution_capability_service import validate_action_execution_mode
from app.services.ops_release_service import (
_calculate_total_batches,
_normalize_rollout_policy,
_resolve_rollout_targets,
build_release_rollout_gate,
get_release,
)
_LOW_RISK_ACTIONS = {
"health.snapshot",
"runtime.push_sync",
"runtime.pull_tasks",
"logs.collect",
"diagnostics.collect",
"delivery.queue.flush",
}
_MEDIUM_RISK_ACTIONS = {
"runtime.start_worker",
"runtime.start_sync_agent",
"service.status",
"health.check",
"delivery.queue.replay",
}
_HIGH_RISK_ACTIONS = {
"runtime.stop_worker",
"runtime.stop_sync_agent",
"runtime.restart_api",
"service.restart",
"deploy.release",
"delivery.queue.discard",
}
_CRITICAL_RISK_ACTIONS = {
"deploy.rollback",
"node.bootstrap",
"cluster.reconfigure",
}
def _normalize_node_code_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n").strip()
if not text:
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def _dedupe_text_items(items: list[str]) -> list[str]:
return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip()))
def _risk_level_for_action(action: str) -> str:
if action in _LOW_RISK_ACTIONS:
return "low"
if action in _HIGH_RISK_ACTIONS:
return "high"
if action in _CRITICAL_RISK_ACTIONS:
return "critical"
return "medium"
def _compact_target_node(item: dict) -> dict:
return {
"node_code": str(item.get("node_code") or ""),
"region": str(item.get("region") or ""),
"role": str(item.get("role") or ""),
"status": str(item.get("status") or ""),
"current_load": int(item.get("current_load", 0) or 0),
"is_effective_worker": bool(item.get("is_effective_worker", False)),
"detect_participating": bool(item.get("detect_participating", False)),
"is_enabled": bool(item.get("is_enabled", True)),
"deploy_channel": str(item.get("deploy_channel") or ""),
}
def _preview_action_payload_guardrails(
action: str,
payload: dict,
*,
execution_mode: str,
target_nodes_total: int,
) -> dict:
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if action == "deploy.release":
release_version = str(payload.get("release_version") or "").strip()
artifact_url = str(payload.get("artifact_url") or "").strip()
if not release_version:
blocking_reasons.append("deploy.release 缺少 release_version当前不能创建正式发布任务。")
if not artifact_url:
blocking_reasons.append("deploy.release 缺少 artifact_url当前不能创建正式发布任务。")
if execution_mode not in {"remote-agent", "ssh"}:
blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。")
if target_nodes_total > 1:
recommendations.append("建议先单节点 canary 验证发布任务,再逐步扩到更多节点。")
if action == "node.bootstrap":
if execution_mode != "control-plane":
blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。")
if target_nodes_total > 1:
recommendations.append("接管动作建议按单节点节奏推进,先确认首台节点接入成功后再继续放量。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
return {
"warnings": _dedupe_text_items(warnings),
"blocking_reasons": _dedupe_text_items(blocking_reasons),
"approval_reasons": _dedupe_text_items(approval_reasons),
"recommendations": _dedupe_text_items(recommendations),
}
def _preview_single_node_policy(
action: str,
*,
target_node_code: str,
cluster: dict,
payload: dict | None = None,
execution_mode: str = "remote-agent",
target_type: str = "node",
include_payload_guardrails: bool = True,
) -> dict:
payload = dict(payload or {})
nodes = list(cluster.get("nodes") or [])
summary = cluster.get("summary") or {}
target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
execution_supported, execution_reason = validate_action_execution_mode(
action,
execution_mode,
target_node_code=target_node_code,
current_node_code=settings.node_code,
)
if not execution_supported and execution_reason:
blocking_reasons.append(execution_reason)
if target_type == "node" and target_node_code and not target_node:
warnings.append("目标节点当前未出现在集群快照中,可能尚未注册或已离线。")
node_status = str(target_node.get("status") or "")
node_role = str(target_node.get("role") or "")
node_current_load = int(target_node.get("current_load", 0) or 0)
node_effective_worker = bool(target_node.get("is_effective_worker", False))
node_detect_participating = bool(target_node.get("detect_participating", False))
if target_node:
if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("目标节点正在参与检测,需先迁移负载或人工确认后再执行。")
if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
approval_reasons.append("目标节点是 control 节点,建议强制走审批或维护窗口。")
if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("当前有效执行节点数不足 2执行该动作可能导致检测能力中断。")
if node_current_load > 0:
warnings.append(f"目标节点当前负载为 {node_current_load},建议优先观察或转移任务。")
if include_payload_guardrails:
payload_guardrails = _preview_action_payload_guardrails(
action,
payload,
execution_mode=execution_mode,
target_nodes_total=1,
)
warnings.extend(payload_guardrails.get("warnings") or [])
blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(payload_guardrails.get("approval_reasons") or [])
recommendations.extend(payload_guardrails.get("recommendations") or [])
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": target_type,
"target_node_code": target_node_code,
"execution_mode": execution_mode,
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_node": {
"node_code": str(target_node.get("node_code") or ""),
"region": str(target_node.get("region") or ""),
"role": node_role,
"status": node_status,
"current_load": node_current_load,
"is_effective_worker": node_effective_worker,
"detect_participating": node_detect_participating,
} if target_node else {},
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict:
target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes"))
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
summary = cluster.get("summary") or {}
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if not target_node_codes:
blocking_reasons.append("当前没有可预检的目标节点。")
shared_guardrails = _preview_action_payload_guardrails(
action,
action_payload,
execution_mode=execution_mode,
target_nodes_total=len(target_node_codes),
)
warnings.extend(shared_guardrails.get("warnings") or [])
blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(shared_guardrails.get("approval_reasons") or [])
recommendations.extend(shared_guardrails.get("recommendations") or [])
node_previews = [
_preview_single_node_policy(
action,
target_node_code=node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type="batch",
include_payload_guardrails=False,
)
for node_code in target_node_codes
]
blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))]
approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))]
warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])]
missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")]
target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"]
if blocked_nodes:
blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断,需先处理后再批量执行。")
if approval_nodes:
approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。")
if warning_nodes:
warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。")
if missing_nodes:
warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。")
if target_control_nodes and len(target_node_codes) > 1:
recommendations.append("建议把 control 节点放到批次尾部,优先在 worker 或空闲节点验证。")
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
return {
"action": action,
"target_type": "batch",
"execution_mode": execution_mode,
"risk_level": _risk_level_for_action(action),
"approval_required": bool(approval_reasons),
"blocked": bool(blocking_reasons),
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": len(target_node_codes),
"blocked_nodes": len(blocked_nodes),
"approval_nodes": len(approval_nodes),
"warning_nodes": len(warning_nodes),
"missing_nodes": len(missing_nodes),
},
"target_nodes": [
{
**(item.get("target_node") or {}),
"node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""),
"risk_level": str(item.get("risk_level") or ""),
"approval_required": bool(item.get("approval_required", False)),
"blocked": bool(item.get("blocked", False)),
"blocking_reasons": list(item.get("blocking_reasons") or []),
"approval_reasons": list(item.get("approval_reasons") or []),
"warnings": list(item.get("warnings") or []),
}
for item in node_previews
],
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict:
selector = payload.get("target_selector") or {}
input_policy = payload.get("policy") or {}
release_id = int(payload.get("release_id") or 0)
input_release = dict(payload.get("release") or {})
release = input_release or (get_release(release_id) if release_id > 0 else {})
target_nodes = _resolve_rollout_targets(selector)
normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes))
execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
summary = cluster.get("summary") or {}
release_gate = build_release_rollout_gate(
release,
target_nodes,
execution_mode=execution_mode,
)
operational_readiness = dict(release_gate.get("operational_readiness") or {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
target_count = len(target_nodes)
first_batch_size = int(normalized_policy.get("first_batch_size") or 0)
batch_size = int(normalized_policy.get("batch_size") or 0)
batches_total = _calculate_total_batches(target_count, normalized_policy)
first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else []
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0)
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"]
target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"]
target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))]
first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))]
first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"]
busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0]
participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))]
offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}]
if target_count <= 0:
blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。")
if release_id > 0 and not input_release and not release:
blocking_reasons.append("指定的 Release 不存在,当前无法预检该 rollout。")
gate_status = str(release_gate.get("status") or "").strip()
if gate_status in {"release_not_ready", "artifact_missing"}:
warnings.append(str(release_gate.get("summary") or "").strip())
if offline_targets:
warnings.append(f"命中了 {len(offline_targets)} 个非在线节点,若保留 only_online=false发布时可能出现排队或失败。")
if busy_targets:
warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点,建议优先确认任务是否可中断。")
if participating_targets:
warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点,建议优先滚动到空闲节点。")
if target_control_nodes:
approval_reasons.append("本次 rollout 覆盖 control 节点,建议强制审批,并把 control 放在最后一批。")
if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes:
blocking_reasons.append("第一批会覆盖全部有效执行节点,检测能力可能瞬时归零。请缩小 first_batch_size。")
if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes:
blocking_reasons.append("第一批会覆盖全部在线控制面,运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。")
if first_batch_size > 1 and target_control_nodes:
warnings.append("当前第一批大于 1且目标中包含 control 节点;默认建议先 worker、后 control。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
blocking_reasons.extend(list(release_gate.get("blocking_reasons") or []))
warnings.extend(list(release_gate.get("warning_reasons") or []))
recommendations.extend(list(release_gate.get("recommendations") or []))
readiness_summary = dict(operational_readiness.get("summary") or {})
if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count:
approval_reasons.append("本次目标中存在巡检未完全通过的节点,建议先完成标准巡检并经人工确认后再发布。")
if target_count > 1 and first_batch_size > 1:
recommendations.append("正式环境建议 first_batch_size=1从单节点 canary 开始。")
if target_control_nodes:
recommendations.append("建议把 control 节点放在 rollout 末尾,并确保 worker 已先通过健康检查。")
if busy_targets or participating_targets:
recommendations.append("建议先等参与检测的节点降为空闲,再推进该批次。")
warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip()))
blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip()))
approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip()))
recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip()))
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": "rollout",
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": target_count,
"worker_nodes": len(target_worker_nodes),
"control_nodes": len(target_control_nodes),
"effective_worker_nodes": len(target_effective_workers),
"busy_nodes": len(busy_targets),
"participating_nodes": len(participating_targets),
"offline_nodes": len(offline_targets),
},
"batch_plan": {
"first_batch_size": first_batch_size,
"batch_size": batch_size,
"batches_total": batches_total,
"first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets],
"remaining_after_first_batch": max(target_count - len(first_batch_targets), 0),
},
"cluster_guardrails": {
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"online_control_nodes": online_control_nodes,
"busy_nodes": list(summary.get("busy_nodes") or []),
},
"release": release,
"release_gate": release_gate,
"target_nodes": [_compact_target_node(item) for item in target_nodes],
"operational_readiness": operational_readiness,
}
def preview_ops_job_policy(payload: dict) -> dict:
action = str(payload.get("action") or "").strip()
target_type = str(payload.get("target_type") or "node").strip() or "node"
target_node_code = str(payload.get("target_node_code") or "").strip()
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
cluster = get_cluster_snapshot()
if target_type == "rollout":
return _preview_rollout_policy(action, payload, cluster)
if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []):
return _preview_node_batch_policy(action, payload, cluster)
return _preview_single_node_policy(
action,
target_node_code=target_node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type=target_type,
include_payload_guardrails=True,
)