from __future__ import annotations from app.core.config import settings from app.services.cluster_runtime_service import get_cluster_snapshot from app.services.ops_execution_capability_service import validate_action_execution_mode from app.services.ops_release_service import ( _calculate_total_batches, _normalize_rollout_policy, _resolve_rollout_targets, build_release_rollout_gate, get_release, ) _LOW_RISK_ACTIONS = { "health.snapshot", "runtime.push_sync", "runtime.pull_tasks", "logs.collect", "diagnostics.collect", "delivery.queue.flush", } _MEDIUM_RISK_ACTIONS = { "runtime.start_worker", "runtime.start_sync_agent", "service.status", "health.check", "delivery.queue.replay", } _HIGH_RISK_ACTIONS = { "runtime.stop_worker", "runtime.stop_sync_agent", "runtime.restart_api", "service.restart", "deploy.release", "delivery.queue.discard", } _CRITICAL_RISK_ACTIONS = { "deploy.rollback", "node.bootstrap", "cluster.reconfigure", } def _normalize_node_code_list(raw_value: object) -> list[str]: if isinstance(raw_value, list): return [str(item).strip() for item in raw_value if str(item).strip()] text = str(raw_value or "").replace("\r", "\n").strip() if not text: return [] normalized = text.replace(",", "\n") return [item.strip() for item in normalized.split("\n") if item.strip()] def _dedupe_text_items(items: list[str]) -> list[str]: return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip())) def _risk_level_for_action(action: str) -> str: if action in _LOW_RISK_ACTIONS: return "low" if action in _HIGH_RISK_ACTIONS: return "high" if action in _CRITICAL_RISK_ACTIONS: return "critical" return "medium" def _compact_target_node(item: dict) -> dict: return { "node_code": str(item.get("node_code") or ""), "region": str(item.get("region") or ""), "role": str(item.get("role") or ""), "status": str(item.get("status") or ""), "current_load": int(item.get("current_load", 0) or 0), "is_effective_worker": bool(item.get("is_effective_worker", False)), "detect_participating": bool(item.get("detect_participating", False)), "is_enabled": bool(item.get("is_enabled", True)), "deploy_channel": str(item.get("deploy_channel") or ""), } def _preview_action_payload_guardrails( action: str, payload: dict, *, execution_mode: str, target_nodes_total: int, ) -> dict: warnings: list[str] = [] blocking_reasons: list[str] = [] approval_reasons: list[str] = [] recommendations: list[str] = [] if action == "deploy.release": release_version = str(payload.get("release_version") or "").strip() artifact_url = str(payload.get("artifact_url") or "").strip() if not release_version: blocking_reasons.append("deploy.release 缺少 release_version，当前不能创建正式发布任务。") if not artifact_url: blocking_reasons.append("deploy.release 缺少 artifact_url，当前不能创建正式发布任务。") if execution_mode not in {"remote-agent", "ssh"}: blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。") if target_nodes_total > 1: recommendations.append("建议先单节点 canary 验证发布任务，再逐步扩到更多节点。") if action == "node.bootstrap": if execution_mode != "control-plane": blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。") if target_nodes_total > 1: recommendations.append("接管动作建议按单节点节奏推进，先确认首台节点接入成功后再继续放量。") if action in _CRITICAL_RISK_ACTIONS: approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。") elif action in _HIGH_RISK_ACTIONS: approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。") return { "warnings": _dedupe_text_items(warnings), "blocking_reasons": _dedupe_text_items(blocking_reasons), "approval_reasons": _dedupe_text_items(approval_reasons), "recommendations": _dedupe_text_items(recommendations), } def _preview_single_node_policy( action: str, *, target_node_code: str, cluster: dict, payload: dict | None = None, execution_mode: str = "remote-agent", target_type: str = "node", include_payload_guardrails: bool = True, ) -> dict: payload = dict(payload or {}) nodes = list(cluster.get("nodes") or []) summary = cluster.get("summary") or {} target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {}) risk_level = _risk_level_for_action(action) warnings: list[str] = [] blocking_reasons: list[str] = [] approval_reasons: list[str] = [] recommendations: list[str] = [] execution_supported, execution_reason = validate_action_execution_mode( action, execution_mode, target_node_code=target_node_code, current_node_code=settings.node_code, ) if not execution_supported and execution_reason: blocking_reasons.append(execution_reason) if target_type == "node" and target_node_code and not target_node: warnings.append("目标节点当前未出现在集群快照中，可能尚未注册或已离线。") node_status = str(target_node.get("status") or "") node_role = str(target_node.get("role") or "") node_current_load = int(target_node.get("current_load", 0) or 0) node_effective_worker = bool(target_node.get("is_effective_worker", False)) node_detect_participating = bool(target_node.get("detect_participating", False)) if target_node: if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: blocking_reasons.append("目标节点当前处于 busy 状态，不适合直接执行中断类动作。") if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: blocking_reasons.append("目标节点正在参与检测，需先迁移负载或人工确认后再执行。") if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: approval_reasons.append("目标节点是 control 节点，建议强制走审批或维护窗口。") if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}: blocking_reasons.append("当前有效执行节点数不足 2，执行该动作可能导致检测能力中断。") if node_current_load > 0: warnings.append(f"目标节点当前负载为 {node_current_load}，建议优先观察或转移任务。") if include_payload_guardrails: payload_guardrails = _preview_action_payload_guardrails( action, payload, execution_mode=execution_mode, target_nodes_total=1, ) warnings.extend(payload_guardrails.get("warnings") or []) blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or []) approval_reasons.extend(payload_guardrails.get("approval_reasons") or []) recommendations.extend(payload_guardrails.get("recommendations") or []) warnings = _dedupe_text_items(warnings) blocking_reasons = _dedupe_text_items(blocking_reasons) approval_reasons = _dedupe_text_items(approval_reasons) recommendations = _dedupe_text_items(recommendations) approval_required = bool(approval_reasons) blocked = bool(blocking_reasons) return { "action": action, "target_type": target_type, "target_node_code": target_node_code, "execution_mode": execution_mode, "risk_level": risk_level, "approval_required": approval_required, "blocked": blocked, "blocking_reasons": blocking_reasons, "approval_reasons": approval_reasons, "warnings": warnings, "recommendations": recommendations, "target_node": { "node_code": str(target_node.get("node_code") or ""), "region": str(target_node.get("region") or ""), "role": node_role, "status": node_status, "current_load": node_current_load, "is_effective_worker": node_effective_worker, "detect_participating": node_detect_participating, } if target_node else {}, "cluster_guardrails": { "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0), "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0), "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0), "busy_nodes": list(summary.get("busy_nodes") or []), }, } def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict: target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes")) execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent" action_payload = dict(payload.get("payload") or {}) summary = cluster.get("summary") or {} warnings: list[str] = [] blocking_reasons: list[str] = [] approval_reasons: list[str] = [] recommendations: list[str] = [] if not target_node_codes: blocking_reasons.append("当前没有可预检的目标节点。") shared_guardrails = _preview_action_payload_guardrails( action, action_payload, execution_mode=execution_mode, target_nodes_total=len(target_node_codes), ) warnings.extend(shared_guardrails.get("warnings") or []) blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or []) approval_reasons.extend(shared_guardrails.get("approval_reasons") or []) recommendations.extend(shared_guardrails.get("recommendations") or []) node_previews = [ _preview_single_node_policy( action, target_node_code=node_code, cluster=cluster, payload=action_payload, execution_mode=execution_mode, target_type="batch", include_payload_guardrails=False, ) for node_code in target_node_codes ] blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))] approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))] warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])] missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")] target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"] if blocked_nodes: blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断，需先处理后再批量执行。") if approval_nodes: approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。") if warning_nodes: warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。") if missing_nodes: warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。") if target_control_nodes and len(target_node_codes) > 1: recommendations.append("建议把 control 节点放到批次尾部，优先在 worker 或空闲节点验证。") warnings = _dedupe_text_items(warnings) blocking_reasons = _dedupe_text_items(blocking_reasons) approval_reasons = _dedupe_text_items(approval_reasons) recommendations = _dedupe_text_items(recommendations) return { "action": action, "target_type": "batch", "execution_mode": execution_mode, "risk_level": _risk_level_for_action(action), "approval_required": bool(approval_reasons), "blocked": bool(blocking_reasons), "blocking_reasons": blocking_reasons, "approval_reasons": approval_reasons, "warnings": warnings, "recommendations": recommendations, "target_summary": { "nodes_total": len(target_node_codes), "blocked_nodes": len(blocked_nodes), "approval_nodes": len(approval_nodes), "warning_nodes": len(warning_nodes), "missing_nodes": len(missing_nodes), }, "target_nodes": [ { **(item.get("target_node") or {}), "node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""), "risk_level": str(item.get("risk_level") or ""), "approval_required": bool(item.get("approval_required", False)), "blocked": bool(item.get("blocked", False)), "blocking_reasons": list(item.get("blocking_reasons") or []), "approval_reasons": list(item.get("approval_reasons") or []), "warnings": list(item.get("warnings") or []), } for item in node_previews ], "cluster_guardrails": { "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0), "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0), "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0), "busy_nodes": list(summary.get("busy_nodes") or []), }, } def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict: selector = payload.get("target_selector") or {} input_policy = payload.get("policy") or {} release_id = int(payload.get("release_id") or 0) input_release = dict(payload.get("release") or {}) release = input_release or (get_release(release_id) if release_id > 0 else {}) target_nodes = _resolve_rollout_targets(selector) normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes)) execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent" summary = cluster.get("summary") or {} release_gate = build_release_rollout_gate( release, target_nodes, execution_mode=execution_mode, ) operational_readiness = dict(release_gate.get("operational_readiness") or {}) risk_level = _risk_level_for_action(action) warnings: list[str] = [] blocking_reasons: list[str] = [] approval_reasons: list[str] = [] recommendations: list[str] = [] target_count = len(target_nodes) first_batch_size = int(normalized_policy.get("first_batch_size") or 0) batch_size = int(normalized_policy.get("batch_size") or 0) batches_total = _calculate_total_batches(target_count, normalized_policy) first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else [] online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0) dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0) online_control_nodes = int(summary.get("online_control_nodes", 0) or 0) target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"] target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"] target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))] first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))] first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"] busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0] participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))] offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}] if target_count <= 0: blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。") if release_id > 0 and not input_release and not release: blocking_reasons.append("指定的 Release 不存在，当前无法预检该 rollout。") gate_status = str(release_gate.get("status") or "").strip() if gate_status in {"release_not_ready", "artifact_missing"}: warnings.append(str(release_gate.get("summary") or "").strip()) if offline_targets: warnings.append(f"命中了 {len(offline_targets)} 个非在线节点，若保留 only_online=false，发布时可能出现排队或失败。") if busy_targets: warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点，建议优先确认任务是否可中断。") if participating_targets: warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点，建议优先滚动到空闲节点。") if target_control_nodes: approval_reasons.append("本次 rollout 覆盖 control 节点，建议强制审批，并把 control 放在最后一批。") if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes: blocking_reasons.append("第一批会覆盖全部有效执行节点，检测能力可能瞬时归零。请缩小 first_batch_size。") if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes: blocking_reasons.append("第一批会覆盖全部在线控制面，运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。") if first_batch_size > 1 and target_control_nodes: warnings.append("当前第一批大于 1，且目标中包含 control 节点；默认建议先 worker、后 control。") if action in _CRITICAL_RISK_ACTIONS: approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。") elif action in _HIGH_RISK_ACTIONS: approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。") blocking_reasons.extend(list(release_gate.get("blocking_reasons") or [])) warnings.extend(list(release_gate.get("warning_reasons") or [])) recommendations.extend(list(release_gate.get("recommendations") or [])) readiness_summary = dict(operational_readiness.get("summary") or {}) if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count: approval_reasons.append("本次目标中存在巡检未完全通过的节点，建议先完成标准巡检并经人工确认后再发布。") if target_count > 1 and first_batch_size > 1: recommendations.append("正式环境建议 first_batch_size=1，从单节点 canary 开始。") if target_control_nodes: recommendations.append("建议把 control 节点放在 rollout 末尾，并确保 worker 已先通过健康检查。") if busy_targets or participating_targets: recommendations.append("建议先等参与检测的节点降为空闲，再推进该批次。") warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip())) blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip())) approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip())) recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip())) approval_required = bool(approval_reasons) blocked = bool(blocking_reasons) return { "action": action, "target_type": "rollout", "risk_level": risk_level, "approval_required": approval_required, "blocked": blocked, "blocking_reasons": blocking_reasons, "approval_reasons": approval_reasons, "warnings": warnings, "recommendations": recommendations, "target_summary": { "nodes_total": target_count, "worker_nodes": len(target_worker_nodes), "control_nodes": len(target_control_nodes), "effective_worker_nodes": len(target_effective_workers), "busy_nodes": len(busy_targets), "participating_nodes": len(participating_targets), "offline_nodes": len(offline_targets), }, "batch_plan": { "first_batch_size": first_batch_size, "batch_size": batch_size, "batches_total": batches_total, "first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets], "remaining_after_first_batch": max(target_count - len(first_batch_targets), 0), }, "cluster_guardrails": { "online_worker_nodes": online_worker_nodes, "dedicated_online_worker_nodes": dedicated_online_worker_nodes, "online_control_nodes": online_control_nodes, "busy_nodes": list(summary.get("busy_nodes") or []), }, "release": release, "release_gate": release_gate, "target_nodes": [_compact_target_node(item) for item in target_nodes], "operational_readiness": operational_readiness, } def preview_ops_job_policy(payload: dict) -> dict: action = str(payload.get("action") or "").strip() target_type = str(payload.get("target_type") or "node").strip() or "node" target_node_code = str(payload.get("target_node_code") or "").strip() execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent" action_payload = dict(payload.get("payload") or {}) cluster = get_cluster_snapshot() if target_type == "rollout": return _preview_rollout_policy(action, payload, cluster) if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []): return _preview_node_batch_policy(action, payload, cluster) return _preview_single_node_policy( action, target_node_code=target_node_code, cluster=cluster, payload=action_payload, execution_mode=execution_mode, target_type=target_type, include_payload_guardrails=True, )