Files
getDomain/domain-api/app/services/ops_policy_service.py

516 lines
24 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
from app.core.config import settings
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.ops_execution_capability_service import validate_action_execution_mode
from app.services.ops_release_service import (
_calculate_total_batches,
_normalize_rollout_policy,
_resolve_rollout_targets,
build_release_rollout_gate,
get_release,
)
_LOW_RISK_ACTIONS = {
"health.snapshot",
"runtime.push_sync",
"runtime.pull_tasks",
"logs.collect",
"diagnostics.collect",
"delivery.queue.flush",
}
_MEDIUM_RISK_ACTIONS = {
"runtime.start_worker",
"runtime.start_sync_agent",
"service.status",
"health.check",
"delivery.queue.replay",
}
_HIGH_RISK_ACTIONS = {
"runtime.stop_worker",
"runtime.stop_sync_agent",
"runtime.restart_api",
"service.restart",
"deploy.release",
"delivery.queue.discard",
}
_CRITICAL_RISK_ACTIONS = {
"deploy.rollback",
"node.bootstrap",
"migration.execute",
"cluster.reconfigure",
"runtime.reset_lab_state",
}
def _normalize_node_code_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n").strip()
if not text:
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def _dedupe_text_items(items: list[str]) -> list[str]:
return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip()))
def _risk_level_for_action(action: str) -> str:
if action in _LOW_RISK_ACTIONS:
return "low"
if action in _HIGH_RISK_ACTIONS:
return "high"
if action in _CRITICAL_RISK_ACTIONS:
return "critical"
return "medium"
def _compact_target_node(item: dict) -> dict:
return {
"node_code": str(item.get("node_code") or ""),
"region": str(item.get("region") or ""),
"role": str(item.get("role") or ""),
"status": str(item.get("status") or ""),
"current_load": int(item.get("current_load", 0) or 0),
"is_effective_worker": bool(item.get("is_effective_worker", False)),
"detect_participating": bool(item.get("detect_participating", False)),
"is_enabled": bool(item.get("is_enabled", True)),
"deploy_channel": str(item.get("deploy_channel") or ""),
}
def _preview_action_payload_guardrails(
action: str,
payload: dict,
*,
execution_mode: str,
target_nodes_total: int,
) -> dict:
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if action == "deploy.release":
release_version = str(payload.get("release_version") or "").strip()
artifact_url = str(payload.get("artifact_url") or "").strip()
if not release_version:
blocking_reasons.append("deploy.release 缺少 release_version当前不能创建正式发布任务。")
if not artifact_url:
blocking_reasons.append("deploy.release 缺少 artifact_url当前不能创建正式发布任务。")
if execution_mode not in {"remote-agent", "ssh"}:
blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。")
if target_nodes_total > 1:
recommendations.append("建议先单节点 canary 验证发布任务,再逐步扩到更多节点。")
if action == "node.bootstrap":
if execution_mode != "control-plane":
blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。")
if target_nodes_total > 1:
recommendations.append("接管动作建议按单节点节奏推进,先确认首台节点接入成功后再继续放量。")
if action == "migration.execute":
if execution_mode != "control-plane":
blocking_reasons.append("migration.execute 仅支持 control-plane 执行方式。")
if bool(payload.get("overwrite_database", False)):
approval_reasons.append("迁移任务包含数据库覆盖,正式环境必须显式确认后再执行。")
recommendations.append("迁移属于长任务,建议通过后台任务窗口持续观察日志与健康检查结果。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
return {
"warnings": _dedupe_text_items(warnings),
"blocking_reasons": _dedupe_text_items(blocking_reasons),
"approval_reasons": _dedupe_text_items(approval_reasons),
"recommendations": _dedupe_text_items(recommendations),
}
def _preview_single_node_policy(
action: str,
*,
target_node_code: str,
cluster: dict,
payload: dict | None = None,
execution_mode: str = "remote-agent",
target_type: str = "node",
include_payload_guardrails: bool = True,
) -> dict:
payload = dict(payload or {})
nodes = list(cluster.get("nodes") or [])
summary = cluster.get("summary") or {}
target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
execution_supported, execution_reason = validate_action_execution_mode(
action,
execution_mode,
target_node_code=target_node_code,
current_node_code=settings.node_code,
)
if not execution_supported and execution_reason:
blocking_reasons.append(execution_reason)
if target_type == "node" and target_node_code and not target_node:
warnings.append("目标节点当前未出现在集群快照中,可能尚未注册或已离线。")
node_status = str(target_node.get("status") or "")
node_role = str(target_node.get("role") or "")
node_current_load = int(target_node.get("current_load", 0) or 0)
node_effective_worker = bool(target_node.get("is_effective_worker", False))
node_detect_participating = bool(target_node.get("detect_participating", False))
if target_node:
interrupt_actions = {"runtime.stop_worker", "runtime.restart_api", "service.restart"}
restart_like_actions = {"service.restart"}
rolling_deploy_actions = {"deploy.release", "deploy.rollback"}
if node_status in {"busy"} and action in interrupt_actions - restart_like_actions:
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
elif node_status in {"busy"} and action in restart_like_actions:
warnings.append("目标节点当前处于 busy 状态,重启会带来瞬时抖动,请确认当前窗口可接受。")
elif node_status in {"busy"} and action in rolling_deploy_actions:
warnings.append("目标节点当前处于 busy 状态,滚动发布会触发服务重启,请确认当前窗口可接受短暂抖动。")
if node_detect_participating and action in interrupt_actions - restart_like_actions:
blocking_reasons.append("目标节点正在参与检测,需先迁移负载或人工确认后再执行。")
elif node_detect_participating and action in restart_like_actions:
warnings.append("目标节点正在参与检测,重启会中断当前任务,请确认剩余节点仍可承接负载。")
elif node_detect_participating and action in rolling_deploy_actions:
warnings.append("目标节点正在参与检测,建议优先采用单节点滚动发布,并确认其余节点仍可承接负载。")
if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
approval_reasons.append("目标节点是 control 节点,建议强制走审批或维护窗口。")
if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("当前有效执行节点数不足 2执行该动作可能导致检测能力中断。")
if node_current_load > 0:
warnings.append(f"目标节点当前负载为 {node_current_load},建议优先观察或转移任务。")
if include_payload_guardrails:
payload_guardrails = _preview_action_payload_guardrails(
action,
payload,
execution_mode=execution_mode,
target_nodes_total=1,
)
warnings.extend(payload_guardrails.get("warnings") or [])
blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(payload_guardrails.get("approval_reasons") or [])
recommendations.extend(payload_guardrails.get("recommendations") or [])
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": target_type,
"target_node_code": target_node_code,
"execution_mode": execution_mode,
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_node": {
"node_code": str(target_node.get("node_code") or ""),
"region": str(target_node.get("region") or ""),
"role": node_role,
"status": node_status,
"current_load": node_current_load,
"is_effective_worker": node_effective_worker,
"detect_participating": node_detect_participating,
} if target_node else {},
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict:
target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes"))
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
summary = cluster.get("summary") or {}
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if not target_node_codes:
blocking_reasons.append("当前没有可预检的目标节点。")
shared_guardrails = _preview_action_payload_guardrails(
action,
action_payload,
execution_mode=execution_mode,
target_nodes_total=len(target_node_codes),
)
warnings.extend(shared_guardrails.get("warnings") or [])
blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(shared_guardrails.get("approval_reasons") or [])
recommendations.extend(shared_guardrails.get("recommendations") or [])
node_previews = [
_preview_single_node_policy(
action,
target_node_code=node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type="batch",
include_payload_guardrails=False,
)
for node_code in target_node_codes
]
blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))]
approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))]
warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])]
missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")]
target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"]
if blocked_nodes:
blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断,需先处理后再批量执行。")
if approval_nodes:
approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。")
if warning_nodes:
warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。")
if missing_nodes:
warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。")
if target_control_nodes and len(target_node_codes) > 1:
recommendations.append("建议把 control 节点放到批次尾部,优先在 worker 或空闲节点验证。")
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
return {
"action": action,
"target_type": "batch",
"execution_mode": execution_mode,
"risk_level": _risk_level_for_action(action),
"approval_required": bool(approval_reasons),
"blocked": bool(blocking_reasons),
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": len(target_node_codes),
"blocked_nodes": len(blocked_nodes),
"approval_nodes": len(approval_nodes),
"warning_nodes": len(warning_nodes),
"missing_nodes": len(missing_nodes),
},
"target_nodes": [
{
**(item.get("target_node") or {}),
"node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""),
"risk_level": str(item.get("risk_level") or ""),
"approval_required": bool(item.get("approval_required", False)),
"blocked": bool(item.get("blocked", False)),
"blocking_reasons": list(item.get("blocking_reasons") or []),
"approval_reasons": list(item.get("approval_reasons") or []),
"warnings": list(item.get("warnings") or []),
}
for item in node_previews
],
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict:
selector = payload.get("target_selector") or {}
input_policy = payload.get("policy") or {}
release_id = int(payload.get("release_id") or 0)
input_release = dict(payload.get("release") or {})
release = input_release or (get_release(release_id) if release_id > 0 else {})
target_nodes = _resolve_rollout_targets(selector)
normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes))
execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
summary = cluster.get("summary") or {}
release_gate = build_release_rollout_gate(
release,
target_nodes,
execution_mode=execution_mode,
)
operational_readiness = dict(release_gate.get("operational_readiness") or {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
target_count = len(target_nodes)
first_batch_size = int(normalized_policy.get("first_batch_size") or 0)
batch_size = int(normalized_policy.get("batch_size") or 0)
batches_total = _calculate_total_batches(target_count, normalized_policy)
first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else []
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0)
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"]
target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"]
target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))]
first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))]
first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"]
busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0]
participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))]
offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}]
if target_count <= 0:
blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。")
if release_id > 0 and not input_release and not release:
blocking_reasons.append("指定的 Release 不存在,当前无法预检该 rollout。")
gate_status = str(release_gate.get("status") or "").strip()
if gate_status in {"release_not_ready", "artifact_missing"}:
warnings.append(str(release_gate.get("summary") or "").strip())
if offline_targets:
warnings.append(f"命中了 {len(offline_targets)} 个非在线节点,若保留 only_online=false发布时可能出现排队或失败。")
if busy_targets:
warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点,建议优先确认任务是否可中断。")
if participating_targets:
warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点,建议优先滚动到空闲节点。")
if target_control_nodes:
approval_reasons.append("本次 rollout 覆盖 control 节点,建议强制审批,并把 control 放在最后一批。")
if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes:
blocking_reasons.append("第一批会覆盖全部有效执行节点,检测能力可能瞬时归零。请缩小 first_batch_size。")
if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes:
blocking_reasons.append("第一批会覆盖全部在线控制面,运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。")
if first_batch_size > 1 and target_control_nodes:
warnings.append("当前第一批大于 1且目标中包含 control 节点;默认建议先 worker、后 control。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
blocking_reasons.extend(list(release_gate.get("blocking_reasons") or []))
warnings.extend(list(release_gate.get("warning_reasons") or []))
recommendations.extend(list(release_gate.get("recommendations") or []))
readiness_summary = dict(operational_readiness.get("summary") or {})
if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count:
approval_reasons.append("本次目标中存在巡检未完全通过的节点,建议先完成标准巡检并经人工确认后再发布。")
if target_count > 1 and first_batch_size > 1:
recommendations.append("正式环境建议 first_batch_size=1从单节点 canary 开始。")
if target_control_nodes:
recommendations.append("建议把 control 节点放在 rollout 末尾,并确保 worker 已先通过健康检查。")
if busy_targets or participating_targets:
recommendations.append("建议先等参与检测的节点降为空闲,再推进该批次。")
warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip()))
blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip()))
approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip()))
recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip()))
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": "rollout",
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": target_count,
"worker_nodes": len(target_worker_nodes),
"control_nodes": len(target_control_nodes),
"effective_worker_nodes": len(target_effective_workers),
"busy_nodes": len(busy_targets),
"participating_nodes": len(participating_targets),
"offline_nodes": len(offline_targets),
},
"batch_plan": {
"first_batch_size": first_batch_size,
"batch_size": batch_size,
"batches_total": batches_total,
"first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets],
"remaining_after_first_batch": max(target_count - len(first_batch_targets), 0),
},
"cluster_guardrails": {
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"online_control_nodes": online_control_nodes,
"busy_nodes": list(summary.get("busy_nodes") or []),
},
"release": release,
"release_gate": release_gate,
"target_nodes": [_compact_target_node(item) for item in target_nodes],
"operational_readiness": operational_readiness,
}
def preview_ops_job_policy(payload: dict) -> dict:
action = str(payload.get("action") or "").strip()
target_type = str(payload.get("target_type") or "node").strip() or "node"
target_node_code = str(payload.get("target_node_code") or "").strip()
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
cluster = get_cluster_snapshot()
if target_type == "rollout":
return _preview_rollout_policy(action, payload, cluster)
if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []):
return _preview_node_batch_policy(action, payload, cluster)
return _preview_single_node_policy(
action,
target_node_code=target_node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type=target_type,
include_payload_guardrails=True,
)