Files
getDomain/domain-api/app/services/ops_template_service.py
2026-04-18 23:52:51 +08:00

921 lines
42 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
from copy import deepcopy
from app.core.config import settings
from app.services.ops_execution_mode_service import decorate_execution_mode_fields, execution_mode_label
def _split_text_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n")
if not text.strip():
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def _coerce_bool(raw_value: object) -> bool:
if isinstance(raw_value, bool):
return raw_value
if isinstance(raw_value, (int, float)):
return bool(raw_value)
normalized = str(raw_value or "").strip().lower()
if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
return True
if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
return False
return bool(normalized)
def _is_missing_value(field_type: str, value: object) -> bool:
if field_type == "text_list":
return not list(value or [])
if field_type == "number":
return value is None
if field_type == "boolean":
return value is None
return str(value or "").strip() == ""
def _service_options() -> list[dict]:
return [
{"label": f"API ({settings.api_service_name})", "value": settings.api_service_name},
{"label": f"Worker ({settings.worker_service_name})", "value": settings.worker_service_name},
{"label": f"Sync Agent ({settings.sync_agent_service_name})", "value": settings.sync_agent_service_name},
{"label": "Node Agent (domaincheck-node-agent)", "value": "domaincheck-node-agent"},
]
def _template_catalog() -> list[dict]:
service_options = _service_options()
return [
{
"group_key": "runtime",
"group_title": "运行时控制",
"group_description": "适合日常按钮化控制 Worker / API / Sync Agent。",
"items": [
{
"key": "runtime.start_worker",
"title": "启动 Worker",
"action": "runtime.start_worker",
"description": "启动目标节点的检测 Worker。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.stop_worker",
"title": "停止 Worker",
"action": "runtime.stop_worker",
"description": "停止目标节点的检测 Worker。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.restart_api",
"title": "重启 API",
"action": "runtime.restart_api",
"description": "重启目标节点的 domaincheck-api 服务。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.start_sync_agent",
"title": "启动 Sync Agent",
"action": "runtime.start_sync_agent",
"description": "启动目标节点的同步代理服务。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.stop_sync_agent",
"title": "停止 Sync Agent",
"action": "runtime.stop_sync_agent",
"description": "停止目标节点的同步代理服务。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "service.restart",
"title": "重启任意服务",
"action": "service.restart",
"description": "对指定 systemd 服务执行 restart。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "默认提供 API / Worker / Sync Agent / Node Agent。",
}
],
},
],
},
{
"group_key": "diagnostics",
"group_title": "巡检与取证",
"group_description": "适合做联调、排障、日志回收和节点健康诊断。",
"items": [
{
"key": "health.snapshot",
"title": "采集健康快照",
"action": "health.snapshot",
"description": "采集目标节点关键 systemd 服务状态。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [],
},
{
"key": "logs.collect",
"title": "收集服务日志",
"action": "logs.collect",
"description": "抓取指定服务最近 journalctl 日志。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "默认抓 Worker排 API 问题时可切到 domaincheck-api。",
},
{
"key": "lines",
"label": "日志行数",
"type": "number",
"required": True,
"default": 120,
"min": 20,
"max": 500,
"help": "journalctl -n 的行数,适合短时排障。",
},
],
},
{
"key": "diagnostics.collect",
"title": "收集诊断包",
"action": "diagnostics.collect",
"description": "打包 API / Worker / Sync Agent / Node Agent 的状态与近期日志。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "lines",
"label": "每个服务日志行数",
"type": "number",
"required": True,
"default": 200,
"min": 20,
"max": 800,
"help": "越大越适合深度排障,但任务返回会更重。",
}
],
},
{
"key": "service.status",
"title": "采集服务状态",
"action": "service.status",
"description": "执行 systemctl status --no-pager -l。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "用于查看特定服务当前状态和最近日志片段。",
}
],
},
],
},
{
"group_key": "delivery_queue",
"group_title": "回执队列治理",
"group_description": "适合统一处理 Node Agent 的 pending / dead-letter 回执队列,而不是手工登机删文件。",
"items": [
{
"key": "delivery.queue.flush",
"title": "立即冲刷回执队列",
"action": "delivery.queue.flush",
"description": "立即触发目标节点 Node Agent 冲刷 pending 回执队列。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "limit",
"label": "本次冲刷上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "限制本次最多冲刷的回执数量,避免一次返回过大。",
}
],
},
{
"key": "delivery.queue.replay",
"title": "重放死信回执",
"action": "delivery.queue.replay",
"description": "把 dead-letter 记录重新放回 pending并可立即触发冲刷。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "record_id",
"label": "单条记录 ID",
"type": "text",
"required": False,
"default": "",
"help": "留空表示按下面的 selector 条件批量匹配。",
},
{
"key": "request_kind",
"label": "记录类型",
"type": "text",
"required": False,
"default": "",
"help": "可选 job_complete / job_event。",
},
{
"key": "detail_code",
"label": "错误码",
"type": "text",
"required": False,
"default": "",
"help": "按 last_detail_code 过滤死信记录。",
},
{
"key": "limit",
"label": "重放上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "批量重放时最多处理多少条死信。",
},
{
"key": "flush_after_replay",
"label": "重放后立即冲刷",
"type": "boolean",
"required": True,
"default": True,
"help": "开启后会把重放回 pending 的记录立即补发一次。",
},
{
"key": "reason",
"label": "重放原因",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可选,便于后续回溯为什么执行此次重放。",
},
],
},
{
"key": "delivery.queue.discard",
"title": "丢弃死信回执",
"action": "delivery.queue.discard",
"description": "把死信移出活动队列,保存到 discarded 归档目录。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "record_id",
"label": "单条记录 ID",
"type": "text",
"required": False,
"default": "",
"help": "留空表示按下面的 selector 条件批量匹配。",
},
{
"key": "request_kind",
"label": "记录类型",
"type": "text",
"required": False,
"default": "",
"help": "可选 job_complete / job_event。",
},
{
"key": "detail_code",
"label": "错误码",
"type": "text",
"required": False,
"default": "",
"help": "按 last_detail_code 过滤死信记录。",
},
{
"key": "limit",
"label": "丢弃上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "批量丢弃时最多处理多少条死信。",
},
{
"key": "reason",
"label": "丢弃原因",
"type": "text",
"required": True,
"default": "",
"wide": True,
"help": "必须说明为何确认这些死信可以被放弃。",
},
],
},
],
},
{
"group_key": "onboarding",
"group_title": "接管与纳管",
"group_description": "适合在控制面生成节点接入工单、统一沉淀首轮接管动作。",
"items": [
{
"key": "node.bootstrap",
"title": "生成节点接入工单",
"action": "node.bootstrap",
"description": "在控制面签发 Node Agent Token并生成 bootstrap env / 脚本 / 一键落地命令。",
"risk_level": "critical",
"default_execution_mode": "control-plane",
"execution_modes": ["control-plane"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "control_plane_base_url",
"label": "控制面地址",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空。留空时由控制面自动回退到当前 API 地址。",
},
{
"key": "root_dir",
"label": "目标安装目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "bootstrap_node_agent.sh 安装根目录。",
},
{
"key": "expires_in_hours",
"label": "Token 有效期(小时)",
"type": "number",
"required": True,
"default": 72,
"min": 1,
"max": 720,
"help": "控制新签发 token 的过期时间。",
},
{
"key": "node_region",
"label": "节点地域提示",
"type": "text",
"required": False,
"default": "",
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
},
{
"key": "node_role",
"label": "节点角色提示",
"type": "text",
"required": False,
"default": "",
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
},
],
},
],
},
{
"group_key": "release",
"group_title": "发布与变更",
"group_description": "适合把单节点灰度、点状修复和手工发布收编到标准 deploy.release 任务。",
"items": [
{
"key": "deploy.release.control",
"title": "发布控制面节点",
"action": "deploy.release",
"description": "面向 controller 节点的标准发布任务,默认会重启 API / Worker / Sync Agent 并执行 API 健康检查。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [
settings.api_service_name,
settings.worker_service_name,
settings.sync_agent_service_name,
],
"wide": True,
"rows": 3,
"help": "每行一个服务。控制面默认重启 API / Worker / Sync Agent。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": ["http://127.0.0.1:8100/health"],
"wide": True,
"rows": 2,
"help": "每行一个 URL。控制面默认探活本机 API /health。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [
settings.api_service_name,
settings.worker_service_name,
settings.sync_agent_service_name,
],
"wide": True,
"rows": 3,
"help": "每行一个 systemd 服务,发布后会检查其 active 状态。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
{
"key": "deploy.release.worker",
"title": "发布 Worker 节点",
"action": "deploy.release",
"description": "面向独立 Worker 节点的标准发布任务,默认只重启 Worker 并校验 Worker 服务状态。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["worker"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [settings.worker_service_name],
"wide": True,
"rows": 2,
"help": "每行一个服务。独立 Worker 默认只重启 Worker。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 2,
"help": "Worker 节点通常可留空;为空时不做 URL 探活。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [settings.worker_service_name],
"wide": True,
"rows": 2,
"help": "每行一个 systemd 服务,默认只校验 Worker active 状态。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
{
"key": "deploy.release.custom",
"title": "发布自定义节点",
"action": "deploy.release",
"description": "完全自定义 deploy.release 参数,适合灰度验证、特殊节点或后续扩展场景。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 3,
"help": "每行一个服务。留空表示只落盘版本,不主动重启任何服务。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 2,
"help": "每行一个 URL。留空表示不做 URL 探活。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 3,
"help": "每行一个 systemd 服务。显式留空时不会自动回落到默认 API 检查。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
],
},
]
def get_ops_action_templates() -> dict:
groups = deepcopy(_template_catalog())
items: list[dict] = []
for group in groups:
group_key = str(group.get("group_key") or "").strip()
group_title = str(group.get("group_title") or "").strip()
group_description = str(group.get("group_description") or "").strip()
normalized_group_items: list[dict] = []
for item in list(group.get("items") or []):
normalized_item = decorate_execution_mode_fields(dict(item))
normalized_item["group_key"] = group_key
normalized_item["group_title"] = group_title
normalized_item["group_description"] = group_description
normalized_group_items.append(normalized_item)
items.append(normalized_item)
group["items"] = normalized_group_items
return {
"groups": groups,
"items": items,
"defaults": {
"requested_by": "web-ui",
"execution_mode": "remote-agent",
"execution_mode_label": execution_mode_label("remote-agent"),
},
}
def get_ops_action_template(template_key: str) -> dict:
normalized_key = str(template_key or "").strip()
if not normalized_key:
return {}
for group in _template_catalog():
group_key = str(group.get("group_key") or "").strip()
group_title = str(group.get("group_title") or "").strip()
group_description = str(group.get("group_description") or "").strip()
for item in list(group.get("items") or []):
if str(item.get("key") or "").strip() == normalized_key:
normalized_item = decorate_execution_mode_fields(deepcopy(item))
normalized_item["group_key"] = group_key
normalized_item["group_title"] = group_title
normalized_item["group_description"] = group_description
return normalized_item
return {}
def build_ops_template_payload(template_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
template = get_ops_action_template(template_key)
if not template:
return False, "动作模板不存在", {}
source_payload = dict(payload or {})
normalized_payload: dict = {}
for field in list(template.get("fields") or []):
field_key = str(field.get("key") or "").strip()
if not field_key:
continue
field_type = str(field.get("type") or "text").strip()
raw_value = source_payload.get(field_key, field.get("default"))
if field_type == "number":
try:
value = int(raw_value or 0)
except Exception:
return False, f"{field_key} 必须是数字", {}
min_value = field.get("min")
max_value = field.get("max")
if min_value is not None:
value = max(int(min_value), value)
if max_value is not None:
value = min(int(max_value), value)
elif field_type == "boolean":
value = _coerce_bool(raw_value)
elif field_type == "text_list":
value = _split_text_list(raw_value)
else:
value = str(raw_value or "").strip()
if bool(field.get("required", False)) and _is_missing_value(field_type, value):
return False, f"{field_key} 不能为空", {}
normalized_payload[field_key] = value
return True, "ok", normalized_payload