2853 lines
123 KiB
Python
2853 lines
123 KiB
Python
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import secrets
|
||
import shlex
|
||
import threading
|
||
from datetime import datetime, timedelta
|
||
|
||
from app.core.config import settings
|
||
from app.core.db import get_db
|
||
from app.services.ops_command_service import build_bash_command
|
||
from app.services.ops_job_service import create_ops_job, ensure_ops_schema, get_ops_job
|
||
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
|
||
|
||
|
||
_AGENT_SCHEMA_SQL = """
|
||
CREATE TABLE IF NOT EXISTS ops_node_tokens (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
token_hash VARCHAR(128) NOT NULL UNIQUE,
|
||
node_code VARCHAR(64) NOT NULL DEFAULT '',
|
||
purpose VARCHAR(32) NOT NULL DEFAULT 'agent',
|
||
issued_by VARCHAR(64) NOT NULL DEFAULT 'system',
|
||
is_enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||
expires_at TIMESTAMP,
|
||
last_used_at TIMESTAMP,
|
||
metadata_json JSONB,
|
||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||
);
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_ops_node_tokens_node_code
|
||
ON ops_node_tokens(node_code, is_enabled, purpose);
|
||
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT '';
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) NOT NULL DEFAULT 'agent';
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS issued_by VARCHAR(64) NOT NULL DEFAULT 'system';
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN NOT NULL DEFAULT TRUE;
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP;
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS last_used_at TIMESTAMP;
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS metadata_json JSONB;
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
|
||
ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
|
||
|
||
CREATE TABLE IF NOT EXISTS ops_job_events (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
job_id BIGINT REFERENCES ops_jobs(id) ON DELETE CASCADE,
|
||
step_id BIGINT REFERENCES ops_job_steps(id) ON DELETE SET NULL,
|
||
node_code VARCHAR(64) NOT NULL DEFAULT '',
|
||
client_event_id VARCHAR(128) NOT NULL DEFAULT '',
|
||
event_type VARCHAR(64) NOT NULL DEFAULT '',
|
||
level VARCHAR(16) NOT NULL DEFAULT 'info',
|
||
message TEXT NOT NULL DEFAULT '',
|
||
payload_json JSONB,
|
||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||
);
|
||
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS job_id BIGINT;
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS step_id BIGINT;
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT '';
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS client_event_id VARCHAR(128) NOT NULL DEFAULT '';
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS event_type VARCHAR(64) NOT NULL DEFAULT '';
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS level VARCHAR(16) NOT NULL DEFAULT 'info';
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS message TEXT NOT NULL DEFAULT '';
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS payload_json JSONB;
|
||
ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP;
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_ops_job_events_job_created
|
||
ON ops_job_events(job_id, created_at DESC);
|
||
|
||
CREATE UNIQUE INDEX IF NOT EXISTS uq_ops_job_events_job_client_event
|
||
ON ops_job_events(job_id, client_event_id)
|
||
WHERE client_event_id <> '';
|
||
|
||
ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS last_agent_complete_request_id VARCHAR(128) NOT NULL DEFAULT '';
|
||
"""
|
||
_OPS_AGENT_SCHEMA_LOCK = threading.Lock()
|
||
_OPS_AGENT_SCHEMA_READY = False
|
||
_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY = 90421802
|
||
|
||
|
||
def ensure_ops_agent_schema() -> None:
|
||
global _OPS_AGENT_SCHEMA_READY
|
||
ensure_ops_schema()
|
||
if _OPS_AGENT_SCHEMA_READY:
|
||
return
|
||
with _OPS_AGENT_SCHEMA_LOCK:
|
||
if _OPS_AGENT_SCHEMA_READY:
|
||
return
|
||
with get_db() as conn:
|
||
conn.autocommit = False
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,))
|
||
cur.execute(_AGENT_SCHEMA_SQL)
|
||
conn.commit()
|
||
_OPS_AGENT_SCHEMA_READY = True
|
||
|
||
|
||
def _hash_token(token: str) -> str:
|
||
return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _decode_json(value: object) -> dict:
|
||
if isinstance(value, dict):
|
||
return value
|
||
if value in (None, ""):
|
||
return {}
|
||
try:
|
||
return json.loads(value)
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _format_time(value: object) -> str:
|
||
if isinstance(value, datetime):
|
||
return value.isoformat(sep=" ", timespec="seconds")
|
||
return ""
|
||
|
||
|
||
def _ops_job_event_level_label(level: object) -> str:
|
||
normalized_level = str(level or "").strip().lower()
|
||
mapping = {
|
||
"debug": "调试",
|
||
"info": "信息",
|
||
"success": "成功",
|
||
"warning": "警告",
|
||
"error": "错误",
|
||
"critical": "严重",
|
||
}
|
||
return mapping.get(normalized_level, normalized_level or "信息")
|
||
|
||
|
||
def _build_ops_job_event_row(row: tuple) -> dict:
|
||
created_at = _format_time(row[9])
|
||
payload = _decode_json(row[8])
|
||
job_id = int(row[1]) if row[1] else 0
|
||
event_id = int(row[0])
|
||
event_type = str(row[5] or "")
|
||
level = str(row[6] or "")
|
||
message = str(row[7] or "")
|
||
occurred_at = str(payload.get("occurred_at") or "").strip() or created_at
|
||
summary_text = str(payload.get("summary_text") or "").strip() or message
|
||
focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {}
|
||
if not focus_ref:
|
||
focus_ref = {
|
||
"kind": "ops_job_event",
|
||
"job_id": job_id,
|
||
"focus_event_id": event_id,
|
||
"event_type": event_type,
|
||
}
|
||
return {
|
||
"id": event_id,
|
||
"event_key": f"job-event:{event_id}",
|
||
"job_id": job_id,
|
||
"step_id": int(row[2]) if row[2] else 0,
|
||
"node_code": str(row[3] or ""),
|
||
"client_event_id": str(row[4] or ""),
|
||
"event_type": event_type,
|
||
"level": level,
|
||
"level_label": _ops_job_event_level_label(level),
|
||
"message": message,
|
||
"summary": message,
|
||
"summary_text": summary_text,
|
||
"payload": payload,
|
||
"has_payload": bool(payload),
|
||
"created_at": created_at,
|
||
"occurred_at": occurred_at,
|
||
"focus_ref": focus_ref,
|
||
"ui_intent": {
|
||
"kind": "job_detail",
|
||
"job_id": job_id,
|
||
"focus_event_id": event_id,
|
||
"event_type": event_type,
|
||
},
|
||
}
|
||
|
||
|
||
def summarize_ops_job_events(events: list[dict], *, job_id: int = 0, limit: int = 50) -> dict:
|
||
level_counts: dict[str, int] = {}
|
||
event_type_counts: dict[str, int] = {}
|
||
node_counts: dict[str, int] = {}
|
||
latest_at = ""
|
||
for event in list(events or []):
|
||
event_level = str(event.get("level") or "").strip() or "info"
|
||
event_type = str(event.get("event_type") or "").strip() or "event"
|
||
node_code = str(event.get("node_code") or "").strip()
|
||
occurred_at = str(event.get("occurred_at") or event.get("created_at") or "").strip()
|
||
level_counts[event_level] = int(level_counts.get(event_level, 0) or 0) + 1
|
||
event_type_counts[event_type] = int(event_type_counts.get(event_type, 0) or 0) + 1
|
||
if node_code:
|
||
node_counts[node_code] = int(node_counts.get(node_code, 0) or 0) + 1
|
||
if occurred_at and occurred_at > latest_at:
|
||
latest_at = occurred_at
|
||
return {
|
||
"job_id": int(job_id or 0),
|
||
"returned_total": len(events or []),
|
||
"level_counts": level_counts,
|
||
"event_type_counts": event_type_counts,
|
||
"node_counts": node_counts,
|
||
"latest_at": latest_at,
|
||
"filters": {
|
||
"limit": int(limit or 50),
|
||
},
|
||
}
|
||
|
||
|
||
def _parse_time(value: object) -> datetime | None:
|
||
if isinstance(value, datetime):
|
||
return value
|
||
raw_value = str(value or "").strip()
|
||
if not raw_value:
|
||
return None
|
||
normalized = raw_value.replace("Z", "+00:00")
|
||
try:
|
||
return datetime.fromisoformat(normalized)
|
||
except Exception:
|
||
pass
|
||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"):
|
||
try:
|
||
return datetime.strptime(raw_value, fmt)
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _agent_error(message: str, detail_code: str, data: dict | None = None) -> tuple[bool, str, dict]:
|
||
payload = dict(data or {})
|
||
payload["detail_code"] = str(detail_code or "").strip()
|
||
return False, message, payload
|
||
|
||
|
||
def _agent_state_from_context(
|
||
*,
|
||
last_seen_at: str,
|
||
latest_token: dict,
|
||
cluster_status: str,
|
||
current_load: int,
|
||
) -> tuple[str, str, str]:
|
||
now = datetime.now()
|
||
last_seen_dt = _parse_time(last_seen_at)
|
||
token_status = str(latest_token.get("token_status") or "absent").strip() or "absent"
|
||
|
||
if last_seen_dt:
|
||
current_time = datetime.now(last_seen_dt.tzinfo) if last_seen_dt.tzinfo else now
|
||
age_seconds = max(0, int((current_time - last_seen_dt).total_seconds()))
|
||
if age_seconds <= 180:
|
||
if cluster_status == "busy" or int(current_load or 0) > 0:
|
||
return "online_busy", "已接管", f"Agent 最近心跳 {age_seconds} 秒前，节点当前负载 {int(current_load or 0)}。"
|
||
return "online", "已接管", f"Agent 最近心跳 {age_seconds} 秒前。"
|
||
return "stale", "心跳过期", f"Agent 最近心跳 {age_seconds} 秒前，已超过在线阈值。"
|
||
|
||
if token_status == "active":
|
||
return "pending_bootstrap", "待接入", "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。"
|
||
if token_status == "expired":
|
||
return "token_expired", "Token 过期", "已签发 Agent Token，但该 Token 已过期。"
|
||
if token_status == "disabled":
|
||
return "token_disabled", "Token 已禁用", "最近一枚 Agent Token 已被禁用，需重新签发。"
|
||
|
||
if cluster_status in {"online", "busy"}:
|
||
return "runtime_only", "仅运行态在线", "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。"
|
||
return "unmanaged", "未接管", "当前节点尚未生成有效 Agent 接入状态。"
|
||
|
||
|
||
def _ssh_access_context(node: dict) -> tuple[bool, str, str]:
|
||
ssh_host = str(node.get("ssh_host") or "").strip()
|
||
ssh_user = str(node.get("ssh_user") or "").strip()
|
||
if ssh_host and ssh_user:
|
||
return True, "SSH 已备好", f"已保存 SSH 入口 {ssh_user}@{ssh_host}，后续可由海外控制面 SSH 执行器接管。"
|
||
if ssh_host and not ssh_user:
|
||
return False, "SSH 待完善", "已填写 SSH 主机，但还缺少 ssh_user。"
|
||
if ssh_user and not ssh_host:
|
||
return False, "SSH 待完善", "已填写 ssh_user，但还缺少 ssh_host。"
|
||
return False, "SSH 未配置", "当前还没有保存 SSH 入口。"
|
||
|
||
|
||
def _remote_access_context(node: dict) -> tuple[str, str, str, bool]:
|
||
is_managed = bool(node.get("is_managed", False))
|
||
is_enabled = bool(node.get("is_enabled", False))
|
||
is_agent_online = bool(node.get("is_agent_online", False))
|
||
ssh_ready, ssh_label, ssh_reason = _ssh_access_context(node)
|
||
|
||
if not is_managed:
|
||
return "unmanaged", "未纳管", "当前节点还没有进入海外控制面的托管清单。", False
|
||
if not is_enabled:
|
||
return "disabled", "已停用", "当前节点已纳管，但被标记为停用。", False
|
||
if is_agent_online and ssh_ready:
|
||
return "hybrid_ready", "Agent + SSH", "Node Agent 已在线，SSH 入口也已备好。", True
|
||
if is_agent_online:
|
||
return "agent_ready", "Agent 在线", "当前节点已经进入标准自动执行器就绪状态。", True
|
||
if ssh_ready:
|
||
return (
|
||
"ssh_ready",
|
||
"SSH 可执行",
|
||
f"{ssh_reason} 当前已可执行日志、诊断和部分服务控制；正式发布与标准 Rollout 仍优先依赖 Node Agent。",
|
||
True,
|
||
)
|
||
return "agent_pending", "待接入", "当前还没有可用 Agent，也没有保存 SSH 入口。", False
|
||
|
||
|
||
def _delivery_queue_context(metadata: dict) -> tuple[str, str, str, dict]:
|
||
snapshot = dict((metadata or {}).get("delivery_queue") or {})
|
||
pending_count = int(snapshot.get("pending_count", 0) or 0)
|
||
dead_letter_count = int(snapshot.get("dead_letter_count", 0) or 0)
|
||
last_flush_at = str(snapshot.get("last_flush_at") or "").strip()
|
||
oldest_pending_at = str(snapshot.get("oldest_pending_at") or "").strip()
|
||
oldest_dead_letter_at = str(snapshot.get("oldest_dead_letter_at") or "").strip()
|
||
|
||
if not snapshot:
|
||
return "unknown", "未上报", "Node Agent 尚未上报回执队列状态。", {}
|
||
if dead_letter_count > 0:
|
||
reason = f"当前存在 {dead_letter_count} 条死信记录，建议优先查看节点日志或诊断编排。"
|
||
if oldest_dead_letter_at:
|
||
reason += f" 最早死信时间：{oldest_dead_letter_at}。"
|
||
return "dead_letter", f"死信 {dead_letter_count}", reason, snapshot
|
||
if pending_count > 0:
|
||
reason = f"当前存在 {pending_count} 条待重试回执，Node Agent 会继续自动回放。"
|
||
if oldest_pending_at:
|
||
reason += f" 最早积压时间：{oldest_pending_at}。"
|
||
return "retrying", f"待重试 {pending_count}", reason, snapshot
|
||
reason = "当前没有待重试回执，也没有死信记录。"
|
||
if last_flush_at:
|
||
reason += f" 最近一次队列冲刷：{last_flush_at}。"
|
||
return "healthy", "正常", reason, snapshot
|
||
|
||
|
||
def _summarize_managed_node_states(nodes: list[dict]) -> dict:
|
||
status_counts: dict[str, int] = {}
|
||
remote_access_counts: dict[str, int] = {}
|
||
delivery_queue_state_counts: dict[str, int] = {}
|
||
for item in nodes:
|
||
state = str(item.get("agent_state") or "").strip() or "unknown"
|
||
status_counts[state] = int(status_counts.get(state, 0) or 0) + 1
|
||
remote_state = str(item.get("remote_access_state") or "").strip() or "unknown"
|
||
remote_access_counts[remote_state] = int(remote_access_counts.get(remote_state, 0) or 0) + 1
|
||
queue_state = str(item.get("delivery_queue_state") or "").strip() or "unknown"
|
||
delivery_queue_state_counts[queue_state] = int(delivery_queue_state_counts.get(queue_state, 0) or 0) + 1
|
||
managed_nodes = [item for item in nodes if bool(item.get("is_managed", False))]
|
||
participating_nodes = [item for item in nodes if bool(item.get("is_current_participant", False))]
|
||
dispatch_active_nodes = [item for item in nodes if bool(item.get("is_dispatch_active", False))]
|
||
recent_only_nodes = [
|
||
item for item in nodes if str(item.get("participation_state") or "").strip() == "recent_throughput"
|
||
]
|
||
standby_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "standby"]
|
||
load_syncing_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "load_syncing"]
|
||
return {
|
||
"status_counts": status_counts,
|
||
"online": int(status_counts.get("online", 0) or 0) + int(status_counts.get("online_busy", 0) or 0),
|
||
"stale": int(status_counts.get("stale", 0) or 0),
|
||
"pending_bootstrap": int(status_counts.get("pending_bootstrap", 0) or 0),
|
||
"runtime_only": int(status_counts.get("runtime_only", 0) or 0),
|
||
"unmanaged": int(status_counts.get("unmanaged", 0) or 0),
|
||
"token_issue": int(status_counts.get("token_expired", 0) or 0) + int(status_counts.get("token_disabled", 0) or 0),
|
||
"ssh_ready": int(remote_access_counts.get("ssh_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0),
|
||
"agent_ready": int(remote_access_counts.get("agent_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0),
|
||
"remote_access_ready": sum(1 for item in nodes if bool(item.get("is_remote_access_ready", False))),
|
||
"managed_total": len(managed_nodes),
|
||
"managed_enabled": sum(1 for item in managed_nodes if bool(item.get("is_enabled", False))),
|
||
"queue_retrying_nodes": sum(
|
||
1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "retrying"
|
||
),
|
||
"queue_dead_letter_nodes": sum(
|
||
1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "dead_letter"
|
||
),
|
||
"queue_pending_records": sum(int(item.get("delivery_queue_pending_count", 0) or 0) for item in managed_nodes),
|
||
"queue_dead_letter_records": sum(
|
||
int(item.get("delivery_queue_dead_letter_count", 0) or 0) for item in managed_nodes
|
||
),
|
||
"participating": len(participating_nodes),
|
||
"dispatch_active": len(dispatch_active_nodes),
|
||
"recent_only": len(recent_only_nodes),
|
||
"standby": len(standby_nodes),
|
||
"load_syncing": len(load_syncing_nodes),
|
||
"non_participating": len(standby_nodes) + len(load_syncing_nodes),
|
||
"total": len(nodes),
|
||
"remote_access_state_counts": remote_access_counts,
|
||
"delivery_queue_state_counts": delivery_queue_state_counts,
|
||
}
|
||
|
||
|
||
def _normalize_detect_participation_rows(payload: dict | None = None) -> dict[str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
if "detect" in normalized_payload and isinstance(normalized_payload.get("detect"), dict):
|
||
normalized_payload = dict(normalized_payload.get("detect") or {})
|
||
|
||
rows: dict[str, dict] = {}
|
||
for collection_key in ("participating_nodes", "non_participating_nodes", "standby_nodes"):
|
||
for item in list(normalized_payload.get(collection_key) or []):
|
||
node_code = str((item or {}).get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
rows[node_code] = dict(item or {})
|
||
return rows
|
||
|
||
|
||
def append_ops_job_event(
|
||
*,
|
||
job_id: int | None,
|
||
step_id: int | None = None,
|
||
node_code: str = "",
|
||
client_event_id: str = "",
|
||
event_type: str,
|
||
message: str,
|
||
level: str = "info",
|
||
payload: dict | None = None,
|
||
) -> int:
|
||
ensure_ops_agent_schema()
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO ops_job_events (
|
||
job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at
|
||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP)
|
||
ON CONFLICT (job_id, client_event_id) WHERE client_event_id <> ''
|
||
DO UPDATE SET
|
||
step_id = COALESCE(EXCLUDED.step_id, ops_job_events.step_id),
|
||
node_code = CASE WHEN EXCLUDED.node_code <> '' THEN EXCLUDED.node_code ELSE ops_job_events.node_code END,
|
||
event_type = EXCLUDED.event_type,
|
||
level = EXCLUDED.level,
|
||
message = EXCLUDED.message,
|
||
payload_json = EXCLUDED.payload_json
|
||
RETURNING id
|
||
""",
|
||
(
|
||
int(job_id) if job_id else None,
|
||
int(step_id) if step_id else None,
|
||
str(node_code or "").strip(),
|
||
str(client_event_id or "").strip()[:128],
|
||
str(event_type or "").strip() or "event",
|
||
str(level or "info").strip() or "info",
|
||
str(message or "").strip()[:2000],
|
||
json.dumps(payload or {}, ensure_ascii=False),
|
||
),
|
||
)
|
||
event_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
return event_id
|
||
|
||
|
||
def list_ops_job_events(job_id: int, limit: int = 50) -> list[dict]:
|
||
ensure_ops_agent_schema()
|
||
safe_limit = min(max(int(limit or 50), 1), 200)
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at
|
||
FROM ops_job_events
|
||
WHERE job_id = %s
|
||
ORDER BY id DESC
|
||
LIMIT %s
|
||
""",
|
||
(int(job_id), safe_limit),
|
||
)
|
||
rows = cur.fetchall()
|
||
return [_build_ops_job_event_row(row) for row in rows]
|
||
|
||
|
||
def get_ops_job_event(event_id: int) -> dict:
|
||
ensure_ops_agent_schema()
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at
|
||
FROM ops_job_events
|
||
WHERE id = %s
|
||
LIMIT 1
|
||
""",
|
||
(int(event_id),),
|
||
)
|
||
row = cur.fetchone()
|
||
return _build_ops_job_event_row(row) if row else {}
|
||
|
||
|
||
def list_ops_job_events_for_jobs(job_ids: list[int], limit: int = 120) -> list[dict]:
|
||
ensure_ops_agent_schema()
|
||
normalized_job_ids = sorted({int(item) for item in list(job_ids or []) if int(item or 0) > 0})
|
||
if not normalized_job_ids:
|
||
return []
|
||
safe_limit = min(max(int(limit or 120), 1), 800)
|
||
placeholders = ", ".join(["%s"] * len(normalized_job_ids))
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
f"""
|
||
SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at
|
||
FROM ops_job_events
|
||
WHERE job_id IN ({placeholders})
|
||
ORDER BY id DESC
|
||
LIMIT %s
|
||
""",
|
||
(*normalized_job_ids, safe_limit),
|
||
)
|
||
rows = cur.fetchall()
|
||
return [_build_ops_job_event_row(row) for row in rows]
|
||
|
||
|
||
def list_managed_nodes_with_agent_state(*, participation_payload: dict | None = None) -> dict:
|
||
from app.services.cluster_runtime_service import get_cluster_snapshot
|
||
from app.services.ops_job_service import list_managed_nodes
|
||
|
||
ensure_ops_agent_schema()
|
||
managed_nodes = list_managed_nodes()
|
||
cluster_nodes = list((get_cluster_snapshot().get("nodes") or []))
|
||
cluster_map = {
|
||
str(item.get("node_code") or "").strip(): item
|
||
for item in cluster_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
detect_participation_map = _normalize_detect_participation_rows(participation_payload)
|
||
if not detect_participation_map:
|
||
try:
|
||
from app.services.runtime_status_service import get_runtime_status
|
||
|
||
runtime_payload = get_runtime_status()
|
||
detect_participation_map = _normalize_detect_participation_rows(runtime_payload.get("detect") or {})
|
||
except Exception:
|
||
detect_participation_map = {}
|
||
|
||
latest_tokens: dict[str, dict] = {}
|
||
latest_jobs: dict[str, dict] = {}
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT DISTINCT ON (node_code)
|
||
node_code, id, issued_by, is_enabled, expires_at, last_used_at, metadata_json, created_at, updated_at
|
||
FROM ops_node_tokens
|
||
WHERE purpose = 'agent'
|
||
ORDER BY node_code ASC, created_at DESC, id DESC
|
||
"""
|
||
)
|
||
for row in cur.fetchall():
|
||
node_code = str(row[0] or "").strip()
|
||
if not node_code:
|
||
continue
|
||
expires_at = _format_time(row[4])
|
||
token_status = "active"
|
||
expires_dt = _parse_time(row[4])
|
||
if not bool(row[3]):
|
||
token_status = "disabled"
|
||
elif expires_dt:
|
||
current_time = datetime.now(expires_dt.tzinfo) if expires_dt.tzinfo else datetime.now()
|
||
if expires_dt < current_time:
|
||
token_status = "expired"
|
||
latest_tokens[node_code] = {
|
||
"record_id": int(row[1]),
|
||
"issued_by": str(row[2] or ""),
|
||
"is_enabled": bool(row[3]),
|
||
"expires_at": expires_at,
|
||
"last_used_at": _format_time(row[5]),
|
||
"metadata": _decode_json(row[6]),
|
||
"created_at": _format_time(row[7]),
|
||
"updated_at": _format_time(row[8]),
|
||
"token_status": token_status,
|
||
}
|
||
|
||
cur.execute(
|
||
"""
|
||
SELECT DISTINCT ON (target_node_code)
|
||
target_node_code, id, job_code, action, status, execution_mode, created_at, updated_at
|
||
FROM ops_jobs
|
||
WHERE target_node_code <> ''
|
||
ORDER BY target_node_code ASC, created_at DESC, id DESC
|
||
"""
|
||
)
|
||
for row in cur.fetchall():
|
||
node_code = str(row[0] or "").strip()
|
||
if not node_code:
|
||
continue
|
||
latest_jobs[node_code] = {
|
||
"id": int(row[1]),
|
||
"job_code": str(row[2] or ""),
|
||
"action": str(row[3] or ""),
|
||
"status": str(row[4] or ""),
|
||
"execution_mode": str(row[5] or ""),
|
||
"created_at": _format_time(row[6]),
|
||
"updated_at": _format_time(row[7]),
|
||
}
|
||
|
||
enriched_nodes: list[dict] = []
|
||
seen_node_codes: set[str] = set()
|
||
for node in managed_nodes:
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
seen_node_codes.add(node_code)
|
||
metadata = dict(node.get("metadata") or {})
|
||
cluster_node = cluster_map.get(node_code, {})
|
||
latest_token = latest_tokens.get(node_code, {})
|
||
latest_job = latest_jobs.get(node_code, {})
|
||
participation_row = dict(detect_participation_map.get(node_code) or {})
|
||
last_seen_at = str(node.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip()
|
||
cluster_status = str(cluster_node.get("status") or metadata.get("cluster_status") or "").strip()
|
||
current_load = int(cluster_node.get("current_load", 0) or 0)
|
||
(
|
||
delivery_queue_state,
|
||
delivery_queue_label,
|
||
delivery_queue_reason,
|
||
delivery_queue_snapshot,
|
||
) = _delivery_queue_context(metadata)
|
||
agent_state, agent_state_label, agent_state_reason = _agent_state_from_context(
|
||
last_seen_at=last_seen_at,
|
||
latest_token=latest_token,
|
||
cluster_status=cluster_status,
|
||
current_load=current_load,
|
||
)
|
||
capabilities = list(metadata.get("capabilities") or [])
|
||
merged_node = {
|
||
**node,
|
||
"is_managed": True,
|
||
"last_seen_at": last_seen_at,
|
||
"agent_state": agent_state,
|
||
"agent_state_label": agent_state_label,
|
||
"agent_state_reason": agent_state_reason,
|
||
"is_agent_online": agent_state in {"online", "online_busy"},
|
||
"capabilities": capabilities,
|
||
"capabilities_count": len(capabilities),
|
||
"agent_version": str(metadata.get("agent_version") or "").strip(),
|
||
"agent_hostname": str(metadata.get("hostname") or "").strip(),
|
||
"agent_ip": str(metadata.get("ip") or "").strip(),
|
||
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
|
||
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
|
||
"cluster_status": cluster_status,
|
||
"cluster_current_load": current_load,
|
||
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or metadata.get("last_heartbeat_at") or "").strip(),
|
||
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", metadata.get("is_effective_worker", False))),
|
||
"cluster_detect_participating": bool(
|
||
participation_row.get("is_current_participant", False)
|
||
or cluster_node.get("detect_participating", metadata.get("detect_participating", False))
|
||
),
|
||
"participation_state": str(participation_row.get("participation_state") or "").strip(),
|
||
"participation_label": str(participation_row.get("participation_label") or "").strip(),
|
||
"participation_reason": str(participation_row.get("participation_reason") or "").strip(),
|
||
"is_current_participant": bool(participation_row.get("is_current_participant", False)),
|
||
"is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)),
|
||
"items_total": int(participation_row.get("items_total", 0) or 0),
|
||
"items_pending": int(participation_row.get("items_pending", 0) or 0),
|
||
"items_claimed": int(participation_row.get("items_claimed", 0) or 0),
|
||
"items_running": int(participation_row.get("items_running", 0) or 0),
|
||
"items_completed": int(participation_row.get("items_completed", 0) or 0),
|
||
"items_failed": int(participation_row.get("items_failed", 0) or 0),
|
||
"processed_recent": int(participation_row.get("processed_recent", 0) or 0),
|
||
"processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0),
|
||
"delivery_queue_state": delivery_queue_state,
|
||
"delivery_queue_label": delivery_queue_label,
|
||
"delivery_queue_reason": delivery_queue_reason,
|
||
"delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0),
|
||
"delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0),
|
||
"delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(),
|
||
"delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(),
|
||
"delivery_queue_oldest_pending_request_id": str(
|
||
delivery_queue_snapshot.get("oldest_pending_request_id") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(),
|
||
"delivery_queue_oldest_dead_letter_at": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_at") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_dead_letter_request_id": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_request_id") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_dead_letter_kind": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_kind") or ""
|
||
).strip(),
|
||
"latest_token": latest_token,
|
||
"latest_job": latest_job,
|
||
}
|
||
ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(merged_node)
|
||
remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context(
|
||
merged_node
|
||
)
|
||
|
||
enriched_nodes.append(
|
||
{
|
||
**merged_node,
|
||
"has_ssh_access": ssh_ready,
|
||
"ssh_access_label": ssh_access_label,
|
||
"ssh_access_reason": ssh_access_reason,
|
||
"remote_access_state": remote_access_state,
|
||
"remote_access_label": remote_access_label,
|
||
"remote_access_reason": remote_access_reason,
|
||
"is_remote_access_ready": is_remote_access_ready,
|
||
}
|
||
)
|
||
|
||
for node_code, cluster_node in cluster_map.items():
|
||
if not node_code or node_code in seen_node_codes:
|
||
continue
|
||
metadata = dict(cluster_node.get("metadata") or {})
|
||
latest_token = latest_tokens.get(node_code, {})
|
||
latest_job = latest_jobs.get(node_code, {})
|
||
participation_row = dict(detect_participation_map.get(node_code) or {})
|
||
cluster_status = str(cluster_node.get("status") or "").strip()
|
||
current_load = int(cluster_node.get("current_load", 0) or 0)
|
||
(
|
||
delivery_queue_state,
|
||
delivery_queue_label,
|
||
delivery_queue_reason,
|
||
delivery_queue_snapshot,
|
||
) = _delivery_queue_context(metadata)
|
||
agent_state, agent_state_label, agent_state_reason = _agent_state_from_context(
|
||
last_seen_at="",
|
||
latest_token=latest_token,
|
||
cluster_status=cluster_status,
|
||
current_load=current_load,
|
||
)
|
||
capabilities = list(metadata.get("capabilities") or [])
|
||
fallback_node = {
|
||
"node_code": node_code,
|
||
"region": str(cluster_node.get("region") or "").strip(),
|
||
"role": str(cluster_node.get("role") or "").strip(),
|
||
"title": str(cluster_node.get("hostname") or node_code).strip(),
|
||
"ssh_host": "",
|
||
"ssh_port": 22,
|
||
"ssh_user": "",
|
||
"auth_mode": "key",
|
||
"deploy_channel": "",
|
||
"is_enabled": False,
|
||
"is_managed": False,
|
||
"metadata": {
|
||
**metadata,
|
||
"imported_from_cluster_fallback": True,
|
||
},
|
||
"created_at": "",
|
||
"last_seen_at": "",
|
||
"updated_at": "",
|
||
"agent_state": agent_state,
|
||
"agent_state_label": agent_state_label,
|
||
"agent_state_reason": agent_state_reason,
|
||
"is_agent_online": agent_state in {"online", "online_busy"},
|
||
"capabilities": capabilities,
|
||
"capabilities_count": len(capabilities),
|
||
"agent_version": str(metadata.get("agent_version") or "").strip(),
|
||
"agent_hostname": str(cluster_node.get("hostname") or metadata.get("hostname") or "").strip(),
|
||
"agent_ip": str(cluster_node.get("ip") or metadata.get("ip") or "").strip(),
|
||
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
|
||
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
|
||
"cluster_status": cluster_status,
|
||
"cluster_current_load": current_load,
|
||
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or "").strip(),
|
||
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", False)),
|
||
"cluster_detect_participating": bool(
|
||
participation_row.get("is_current_participant", False) or cluster_node.get("detect_participating", False)
|
||
),
|
||
"participation_state": str(participation_row.get("participation_state") or "").strip(),
|
||
"participation_label": str(participation_row.get("participation_label") or "").strip(),
|
||
"participation_reason": str(participation_row.get("participation_reason") or "").strip(),
|
||
"is_current_participant": bool(participation_row.get("is_current_participant", False)),
|
||
"is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)),
|
||
"items_total": int(participation_row.get("items_total", 0) or 0),
|
||
"items_pending": int(participation_row.get("items_pending", 0) or 0),
|
||
"items_claimed": int(participation_row.get("items_claimed", 0) or 0),
|
||
"items_running": int(participation_row.get("items_running", 0) or 0),
|
||
"items_completed": int(participation_row.get("items_completed", 0) or 0),
|
||
"items_failed": int(participation_row.get("items_failed", 0) or 0),
|
||
"processed_recent": int(participation_row.get("processed_recent", 0) or 0),
|
||
"processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0),
|
||
"delivery_queue_state": delivery_queue_state,
|
||
"delivery_queue_label": delivery_queue_label,
|
||
"delivery_queue_reason": delivery_queue_reason,
|
||
"delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0),
|
||
"delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0),
|
||
"delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(),
|
||
"delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(),
|
||
"delivery_queue_oldest_pending_request_id": str(
|
||
delivery_queue_snapshot.get("oldest_pending_request_id") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(),
|
||
"delivery_queue_oldest_dead_letter_at": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_at") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_dead_letter_request_id": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_request_id") or ""
|
||
).strip(),
|
||
"delivery_queue_oldest_dead_letter_kind": str(
|
||
delivery_queue_snapshot.get("oldest_dead_letter_kind") or ""
|
||
).strip(),
|
||
"latest_token": latest_token,
|
||
"latest_job": latest_job,
|
||
}
|
||
ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(fallback_node)
|
||
remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context(
|
||
fallback_node
|
||
)
|
||
enriched_nodes.append(
|
||
{
|
||
**fallback_node,
|
||
"has_ssh_access": ssh_ready,
|
||
"ssh_access_label": ssh_access_label,
|
||
"ssh_access_reason": ssh_access_reason,
|
||
"remote_access_state": remote_access_state,
|
||
"remote_access_label": remote_access_label,
|
||
"remote_access_reason": remote_access_reason,
|
||
"is_remote_access_ready": is_remote_access_ready,
|
||
}
|
||
)
|
||
|
||
enriched_nodes.sort(
|
||
key=lambda item: (
|
||
0 if bool(item.get("is_managed", False)) else 1,
|
||
str(item.get("region") or ""),
|
||
str(item.get("role") or ""),
|
||
str(item.get("node_code") or ""),
|
||
)
|
||
)
|
||
|
||
return {
|
||
"nodes": enriched_nodes,
|
||
"summary": _summarize_managed_node_states(enriched_nodes),
|
||
}
|
||
|
||
|
||
def _build_managed_node_handover_stage(node: dict) -> dict:
|
||
normalized_node = dict(node or {})
|
||
node_code = str(normalized_node.get("node_code") or "").strip()
|
||
agent_state = str(normalized_node.get("agent_state") or "").strip()
|
||
agent_state_label = str(normalized_node.get("agent_state_label") or "").strip() or "未知"
|
||
agent_state_reason = str(normalized_node.get("agent_state_reason") or "").strip()
|
||
remote_access_state = str(normalized_node.get("remote_access_state") or "").strip()
|
||
remote_access_label = str(normalized_node.get("remote_access_label") or "").strip() or "未就绪"
|
||
remote_access_reason = str(normalized_node.get("remote_access_reason") or "").strip()
|
||
token_status = str(((normalized_node.get("latest_token") or {}).get("token_status") or "")).strip()
|
||
is_managed = bool(normalized_node.get("is_managed", False))
|
||
is_enabled = bool(normalized_node.get("is_enabled", False))
|
||
has_ssh_access = bool(normalized_node.get("has_ssh_access", False))
|
||
|
||
if not is_managed:
|
||
return {
|
||
"code": "unmanaged",
|
||
"label": "未纳管",
|
||
"summary": "节点已出现在集群视图，但还没保存到海外控制面的托管清单。",
|
||
"next_step": {
|
||
"code": "save_managed_node",
|
||
"label": "先保存节点资料",
|
||
"summary": "先补齐 SSH 入口、身份和部署通道，再生成 Node Agent 接入方案。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_edit",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if not is_enabled:
|
||
return {
|
||
"code": "disabled",
|
||
"label": "已停用",
|
||
"summary": "节点虽然已经纳管，但当前被标记为停用，标准运维动作不会继续下发。",
|
||
"next_step": {
|
||
"code": "enable_managed_node",
|
||
"label": "检查并重新启用",
|
||
"summary": "确认该节点仍应参与运维后，先在托管节点面板重新启用。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_edit",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if agent_state in {"online", "online_busy"}:
|
||
return {
|
||
"code": "ready",
|
||
"label": "接管完成",
|
||
"summary": "Node Agent 已经在线，节点已经进入标准自动执行链路。",
|
||
"next_step": {
|
||
"code": "run_acceptance",
|
||
"label": "执行接管验收",
|
||
"summary": "建议立刻跑一轮 onboarding.acceptance 或标准巡检，确认日志、健康快照和发布链路都打通。",
|
||
"ui_intent": {
|
||
"kind": "open_playbook_dialog",
|
||
"playbook_key": "onboarding.acceptance",
|
||
"target_node_codes": [node_code] if node_code else [],
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": True,
|
||
},
|
||
},
|
||
}
|
||
if agent_state == "stale":
|
||
return {
|
||
"code": "stale",
|
||
"label": "心跳过期",
|
||
"summary": "节点曾经接入过，但 Node Agent 心跳已经过期，当前不能视为稳定可控。",
|
||
"next_step": {
|
||
"code": "recover_agent",
|
||
"label": "优先恢复 Agent",
|
||
"summary": "先检查 domaincheck-node-agent 服务、网络连通性和控制面地址，再决定是否重新签发接入方案。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_handover",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if token_status in {"expired", "disabled"} or agent_state in {"token_expired", "token_disabled"}:
|
||
return {
|
||
"code": "token_issue",
|
||
"label": "Token 失效",
|
||
"summary": "最近一枚 Node Agent Token 已不可继续使用，需要重新生成接入方案。",
|
||
"next_step": {
|
||
"code": "reissue_bootstrap_plan",
|
||
"label": "重新签发接入方案",
|
||
"summary": "重新生成 bootstrap env 和一键落地片段，再到目标节点执行。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_handover",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if agent_state == "pending_bootstrap":
|
||
return {
|
||
"code": "pending_bootstrap",
|
||
"label": "待执行接入",
|
||
"summary": "控制面已经签发了有效 Token，当前卡在目标节点尚未执行 bootstrap 片段。",
|
||
"next_step": {
|
||
"code": "execute_bootstrap_plan",
|
||
"label": "去目标机执行接入片段",
|
||
"summary": "把 env、脚本或一键落地片段拷到目标机执行，然后观察 register / heartbeat 是否建立。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_handover",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if remote_access_state == "ssh_ready" or has_ssh_access:
|
||
return {
|
||
"code": "ssh_ready",
|
||
"label": remote_access_label or "SSH 已备好",
|
||
"summary": "SSH 入口已经可用，可以从海外控制面统一生成并下发 Node Agent 接入方案。",
|
||
"next_step": {
|
||
"code": "issue_bootstrap_plan",
|
||
"label": "生成接入方案",
|
||
"summary": "下一步就是签发 bootstrap env、脚本和一键落地命令，然后在目标节点启动 domaincheck-node-agent。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_handover",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
if agent_state == "runtime_only":
|
||
return {
|
||
"code": "runtime_only",
|
||
"label": agent_state_label or "仅运行态在线",
|
||
"summary": "控制面能看到 runtime 心跳，但 Node Agent 还没进入标准接管链路。",
|
||
"next_step": {
|
||
"code": "complete_ssh_profile",
|
||
"label": "补齐 SSH 入口并生成接入方案",
|
||
"summary": "先把 SSH 主机和账号补齐，再从海外控制面统一签发 Node Agent 接入方案。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_edit",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
return {
|
||
"code": "profile_incomplete",
|
||
"label": remote_access_label or agent_state_label or "待完善",
|
||
"summary": remote_access_reason or agent_state_reason or "当前接管资料还不完整，尚不能进入标准自动执行链路。",
|
||
"next_step": {
|
||
"code": "complete_ssh_profile",
|
||
"label": "补齐节点接入资料",
|
||
"summary": "至少补齐 SSH 入口，再生成标准 Node Agent 接入方案。",
|
||
"ui_intent": {
|
||
"kind": "managed_node_edit",
|
||
"node_code": node_code,
|
||
},
|
||
},
|
||
}
|
||
|
||
|
||
def _build_managed_node_handover_payload(
|
||
*,
|
||
node: dict,
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
) -> dict:
|
||
normalized_node = dict(node or {})
|
||
normalized_node_code = str(normalized_node.get("node_code") or "").strip()
|
||
if not normalized_node_code:
|
||
return {}
|
||
|
||
node = normalized_node
|
||
metadata = dict(node.get("metadata") or {})
|
||
latest_token = dict(node.get("latest_token") or {})
|
||
latest_job = dict(node.get("latest_job") or {})
|
||
control_plane_value = _normalize_control_plane_base_url(control_plane_base_url)
|
||
if not control_plane_value:
|
||
control_plane_value = (
|
||
_normalize_control_plane_base_url(str(metadata.get("ops_control_plane_base_url") or ""))
|
||
or f"http://127.0.0.1:{settings.api_port}"
|
||
)
|
||
resolved_root_dir = (
|
||
str(root_dir or metadata.get("root_dir") or metadata.get("deploy_root_dir") or "/opt/domaincheck").strip()
|
||
or "/opt/domaincheck"
|
||
)
|
||
|
||
stage = _build_managed_node_handover_stage(node)
|
||
|
||
blocking_items: list[str] = []
|
||
attention_items: list[str] = []
|
||
|
||
if not bool(node.get("is_managed", False)):
|
||
blocking_items.append("当前节点还未纳入海外控制面的托管清单。")
|
||
if bool(node.get("is_managed", False)) and not bool(node.get("is_enabled", False)):
|
||
blocking_items.append("当前节点已纳管，但被标记为停用。")
|
||
if not bool(node.get("has_ssh_access", False)):
|
||
attention_items.append("当前还没有完整 SSH 入口，无法从海外控制面直接接管该节点。")
|
||
|
||
agent_state = str(node.get("agent_state") or "").strip()
|
||
if agent_state == "pending_bootstrap":
|
||
blocking_items.append("已签发有效 Token，但目标节点尚未执行 bootstrap 片段。")
|
||
elif agent_state == "stale":
|
||
blocking_items.append("Node Agent 心跳已过期，需先恢复服务或重新接入。")
|
||
elif agent_state in {"token_expired", "token_disabled"}:
|
||
blocking_items.append("最近一枚 Node Agent Token 已失效，需要重新签发。")
|
||
|
||
if str(node.get("delivery_queue_state") or "").strip() == "dead_letter":
|
||
attention_items.append(
|
||
f"当前节点存在 {int(node.get('delivery_queue_dead_letter_count', 0) or 0)} 条死信回执，建议先查看 delivery queue。"
|
||
)
|
||
elif str(node.get("delivery_queue_state") or "").strip() == "retrying":
|
||
attention_items.append(
|
||
f"当前节点还有 {int(node.get('delivery_queue_pending_count', 0) or 0)} 条待重试回执，Node Agent 会继续自动回放。"
|
||
)
|
||
|
||
return {
|
||
"node_code": normalized_node_code,
|
||
"summary": str(stage.get("summary") or "").strip(),
|
||
"stage": stage,
|
||
"node": node,
|
||
"control_plane_base_url": control_plane_value,
|
||
"root_dir": resolved_root_dir,
|
||
"readiness": {
|
||
"is_managed": bool(node.get("is_managed", False)),
|
||
"is_enabled": bool(node.get("is_enabled", False)),
|
||
"has_ssh_access": bool(node.get("has_ssh_access", False)),
|
||
"is_agent_online": bool(node.get("is_agent_online", False)),
|
||
"is_remote_access_ready": bool(node.get("is_remote_access_ready", False)),
|
||
},
|
||
"ssh_profile": {
|
||
"host": str(node.get("ssh_host") or "").strip(),
|
||
"port": int(node.get("ssh_port", 22) or 22),
|
||
"user": str(node.get("ssh_user") or "").strip(),
|
||
"auth_mode": str(node.get("auth_mode") or "").strip(),
|
||
"label": str(node.get("ssh_access_label") or "").strip(),
|
||
"reason": str(node.get("ssh_access_reason") or "").strip(),
|
||
"ssh_command_hint": (
|
||
f"ssh -p {int(node.get('ssh_port', 22) or 22)} "
|
||
f"{str(node.get('ssh_user') or '').strip()}@{str(node.get('ssh_host') or '').strip()}"
|
||
if str(node.get("ssh_host") or "").strip() and str(node.get("ssh_user") or "").strip()
|
||
else ""
|
||
),
|
||
},
|
||
"cluster": {
|
||
"status": str(node.get("cluster_status") or "").strip(),
|
||
"current_load": int(node.get("cluster_current_load", 0) or 0),
|
||
"last_heartbeat_at": str(node.get("cluster_last_heartbeat_at") or "").strip(),
|
||
"hostname": str(node.get("cluster_hostname") or "").strip(),
|
||
"ip": str(node.get("cluster_ip") or "").strip(),
|
||
"is_effective_worker": bool(node.get("cluster_is_effective_worker", False)),
|
||
},
|
||
"participation": {
|
||
"state": str(node.get("participation_state") or "").strip(),
|
||
"label": str(node.get("participation_label") or "").strip(),
|
||
"reason": str(node.get("participation_reason") or "").strip(),
|
||
"is_current_participant": bool(node.get("is_current_participant", False)),
|
||
"is_dispatch_active": bool(node.get("is_dispatch_active", False)),
|
||
"items_claimed": int(node.get("items_claimed", 0) or 0),
|
||
"items_running": int(node.get("items_running", 0) or 0),
|
||
"items_completed": int(node.get("items_completed", 0) or 0),
|
||
"processed_recent": int(node.get("processed_recent", 0) or 0),
|
||
"processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
|
||
},
|
||
"agent_token": latest_token,
|
||
"latest_job": latest_job,
|
||
"delivery_queue": {
|
||
"state": str(node.get("delivery_queue_state") or "").strip(),
|
||
"label": str(node.get("delivery_queue_label") or "").strip(),
|
||
"reason": str(node.get("delivery_queue_reason") or "").strip(),
|
||
"pending_count": int(node.get("delivery_queue_pending_count", 0) or 0),
|
||
"dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0),
|
||
"last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(),
|
||
},
|
||
"blocking_items": blocking_items,
|
||
"attention_items": attention_items,
|
||
"bootstrap_defaults": {
|
||
"node_code": normalized_node_code,
|
||
"node_region": str(node.get("region") or "mainland").strip() or "mainland",
|
||
"node_role": str(node.get("role") or "worker").strip() or "worker",
|
||
"control_plane_base_url": control_plane_value,
|
||
"root_dir": resolved_root_dir,
|
||
"expires_in_hours": 72,
|
||
"issued_by": "api",
|
||
"metadata": {
|
||
"issued_from": "ops-managed-node-handover",
|
||
"node_region": str(node.get("region") or "mainland").strip() or "mainland",
|
||
"node_role": str(node.get("role") or "worker").strip() or "worker",
|
||
"remote_access_state": str(node.get("remote_access_state") or "").strip(),
|
||
"agent_state": agent_state,
|
||
},
|
||
},
|
||
"endpoints": {
|
||
"detail": f"/api/v1/ops/nodes/{normalized_node_code}/handover",
|
||
"bootstrap_plan": f"/api/v1/ops/nodes/{normalized_node_code}/handover/bootstrap-plan",
|
||
},
|
||
}
|
||
|
||
|
||
def get_managed_node_handover(
|
||
node_code: str,
|
||
*,
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return {}
|
||
|
||
resolved_nodes_payload = dict(nodes_payload or {})
|
||
if not resolved_nodes_payload:
|
||
resolved_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=participation_payload)
|
||
nodes = list(resolved_nodes_payload.get("nodes") or [])
|
||
node = next(
|
||
(dict(item or {}) for item in nodes if str(item.get("node_code") or "").strip() == normalized_node_code),
|
||
{},
|
||
)
|
||
if not node:
|
||
return {}
|
||
return _build_managed_node_handover_payload(
|
||
node=node,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
)
|
||
|
||
|
||
def get_managed_node_onboarding(
|
||
node_code: str,
|
||
*,
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> dict:
|
||
handover = get_managed_node_handover(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not handover:
|
||
return {}
|
||
|
||
node = dict(handover.get("node") or {})
|
||
stage = dict(handover.get("stage") or {})
|
||
readiness = dict(handover.get("readiness") or {})
|
||
normalized_node_code = str(handover.get("node_code") or node_code or "").strip()
|
||
stage_code = str(stage.get("code") or "").strip()
|
||
has_ssh_access = bool(readiness.get("has_ssh_access", False))
|
||
is_agent_online = bool(readiness.get("is_agent_online", False))
|
||
is_remote_access_ready = bool(readiness.get("is_remote_access_ready", False))
|
||
|
||
acceptance_execution_mode = ""
|
||
acceptance_status_code = "blocked"
|
||
acceptance_status_label = "暂不可验收"
|
||
acceptance_summary = "当前还没进入标准接管就绪状态，暂不建议直接跑 onboarding.acceptance。"
|
||
if is_agent_online:
|
||
acceptance_execution_mode = "remote-agent"
|
||
acceptance_status_code = "ready"
|
||
acceptance_status_label = "可远端验收"
|
||
acceptance_summary = "Node Agent 已在线，可以直接跑 onboarding.acceptance 做标准接管验收。"
|
||
elif has_ssh_access:
|
||
acceptance_execution_mode = "ssh"
|
||
acceptance_status_code = "ssh_only"
|
||
acceptance_status_label = "可走 SSH 验收"
|
||
acceptance_summary = "当前 Agent 还没在线，但 SSH 已备好，可先走 SSH 模式做接管验收。"
|
||
|
||
onboarding_stage_code = stage_code or "unknown"
|
||
onboarding_stage_label = str(stage.get("label") or "").strip() or "待判断"
|
||
onboarding_summary = str(handover.get("summary") or stage.get("summary") or "").strip()
|
||
if acceptance_status_code in {"ready", "ssh_only"} and stage_code == "ready":
|
||
onboarding_stage_code = "acceptance_ready"
|
||
onboarding_stage_label = "待接管验收"
|
||
onboarding_summary = acceptance_summary
|
||
|
||
acceptance_payload = {
|
||
"playbook_key": "onboarding.acceptance",
|
||
"target_node_codes": [normalized_node_code] if normalized_node_code else [],
|
||
"execution_mode": acceptance_execution_mode or "remote-agent",
|
||
"auto_approve": True,
|
||
}
|
||
bootstrap_payload = {
|
||
"node_code": normalized_node_code,
|
||
"node_region": str(node.get("region") or "mainland").strip() or "mainland",
|
||
"node_role": str(node.get("role") or "worker").strip() or "worker",
|
||
"control_plane_base_url": str(handover.get("control_plane_base_url") or "").strip(),
|
||
"root_dir": str(handover.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck",
|
||
}
|
||
next_actions = []
|
||
if stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale"}:
|
||
next_actions.append(
|
||
{
|
||
"key": "bootstrap_plan",
|
||
"label": "生成接入方案",
|
||
"summary": "先生成并执行 Node Agent bootstrap 方案，再观察 register / heartbeat 是否建立。",
|
||
"command_hint": (
|
||
build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", "http://127.0.0.1:8100", normalized_node_code)
|
||
if normalized_node_code
|
||
else ""
|
||
),
|
||
}
|
||
)
|
||
next_actions.append(
|
||
{
|
||
"key": "bootstrap_run",
|
||
"label": "签发接入工单",
|
||
"summary": "把本节点 bootstrap 收编进 ops playbook / ops job，形成标准可追踪的纳管回执。",
|
||
"command_hint": (
|
||
build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code)
|
||
if normalized_node_code
|
||
else ""
|
||
),
|
||
}
|
||
)
|
||
if acceptance_status_code in {"ready", "ssh_only"}:
|
||
next_actions.append(
|
||
{
|
||
"key": "run_acceptance",
|
||
"label": "执行接管验收",
|
||
"summary": acceptance_summary,
|
||
"command_hint": (
|
||
"通过 driver / runbook 执行 onboarding.acceptance，确认 health / node-agent / worker 三段验收全部通过。"
|
||
),
|
||
}
|
||
)
|
||
|
||
recovery_action = "noop"
|
||
recovery_label = "当前无需额外恢复动作"
|
||
recovery_summary = onboarding_summary or "当前节点暂无需要执行的接管恢复动作。"
|
||
recovery_command_hint = ""
|
||
recovery_needs_confirmation = True
|
||
recovery_window = "none"
|
||
if onboarding_stage_code == "acceptance_ready" or is_agent_online:
|
||
recovery_action = "run_acceptance"
|
||
recovery_label = "执行接管验收"
|
||
recovery_summary = acceptance_summary or "当前节点已经进入验收窗口，建议直接执行 onboarding.acceptance。"
|
||
recovery_command_hint = (
|
||
build_bash_command("drive_ops_center.sh", "node-acceptance-run", "http://127.0.0.1:8100", normalized_node_code, "cli")
|
||
if normalized_node_code
|
||
else ""
|
||
)
|
||
recovery_window = "acceptance"
|
||
elif onboarding_stage_code in {
|
||
"pending_bootstrap",
|
||
"runtime_only",
|
||
"profile_incomplete",
|
||
"ssh_ready",
|
||
"token_issue",
|
||
"stale",
|
||
"agent_pending",
|
||
"unknown",
|
||
"",
|
||
}:
|
||
recovery_action = "bootstrap_run"
|
||
recovery_label = "签发接入工单"
|
||
recovery_summary = "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。"
|
||
recovery_command_hint = (
|
||
build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code, "cli")
|
||
if normalized_node_code
|
||
else ""
|
||
)
|
||
recovery_window = "bootstrap"
|
||
|
||
return {
|
||
"node_code": normalized_node_code,
|
||
"summary": onboarding_summary,
|
||
"onboarding_stage": {
|
||
"code": onboarding_stage_code,
|
||
"label": onboarding_stage_label,
|
||
"summary": onboarding_summary,
|
||
},
|
||
"handover": handover,
|
||
"acceptance": {
|
||
"status_code": acceptance_status_code,
|
||
"status_label": acceptance_status_label,
|
||
"summary": acceptance_summary,
|
||
"execution_mode": acceptance_execution_mode,
|
||
"is_agent_online": is_agent_online,
|
||
"has_ssh_access": has_ssh_access,
|
||
"is_remote_access_ready": is_remote_access_ready,
|
||
"playbook": acceptance_payload,
|
||
},
|
||
"recommended_actions": next_actions,
|
||
"recovery_decision": {
|
||
"action": recovery_action,
|
||
"label": recovery_label,
|
||
"summary": recovery_summary,
|
||
"command_hint": recovery_command_hint,
|
||
"needs_confirmation": recovery_needs_confirmation,
|
||
"window": recovery_window,
|
||
"stage_code": onboarding_stage_code,
|
||
"acceptance_status_code": acceptance_status_code,
|
||
},
|
||
"bootstrap_plan_request": bootstrap_payload,
|
||
}
|
||
|
||
|
||
def preview_managed_node_onboarding_bootstrap(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip()
|
||
bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {})
|
||
normalized_requested_by = str(requested_by or "api").strip() or "api"
|
||
|
||
from app.services.ops_playbook_service import preview_ops_playbook
|
||
|
||
ok, message, data = preview_ops_playbook(
|
||
{
|
||
"playbook_key": "onboarding.bootstrap",
|
||
"node_codes": [normalized_node_code] if normalized_node_code else [],
|
||
"execution_mode": "control-plane",
|
||
"auto_approve": True,
|
||
"requested_by": normalized_requested_by,
|
||
"payload": bootstrap_plan_request,
|
||
}
|
||
)
|
||
follow_up = {
|
||
"next_stage_code": "acceptance_ready",
|
||
"summary": (
|
||
str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip()
|
||
or "接入工单预览已生成；正式执行后应继续观察 Node Agent register / heartbeat，并确认 onboarding_stage 是否进入 acceptance_ready。"
|
||
),
|
||
"recommended_commands": [
|
||
build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code),
|
||
] if normalized_node_code else [],
|
||
"focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})),
|
||
}
|
||
return ok, message, {
|
||
"node_code": normalized_node_code,
|
||
"requested_by": normalized_requested_by,
|
||
"execution_mode": "control-plane",
|
||
"bootstrap_plan_request": bootstrap_plan_request,
|
||
"onboarding": onboarding,
|
||
"playbook_preview": data,
|
||
"follow_up": follow_up,
|
||
}
|
||
|
||
|
||
def execute_managed_node_onboarding_bootstrap(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip()
|
||
bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {})
|
||
normalized_requested_by = str(requested_by or "api").strip() or "api"
|
||
|
||
from app.services.ops_playbook_service import execute_ops_playbook
|
||
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": "onboarding.bootstrap",
|
||
"node_codes": [normalized_node_code] if normalized_node_code else [],
|
||
"execution_mode": "control-plane",
|
||
"auto_approve": True,
|
||
"requested_by": normalized_requested_by,
|
||
"payload": bootstrap_plan_request,
|
||
}
|
||
)
|
||
follow_up = {
|
||
"next_stage_code": "acceptance_ready",
|
||
"summary": (
|
||
str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip()
|
||
or "接入工单已签发；接下来应继续观察 Node Agent register / heartbeat，并确认 onboarding_stage 是否进入 acceptance_ready。"
|
||
),
|
||
"recommended_commands": [
|
||
build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code),
|
||
] if normalized_node_code else [],
|
||
"focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})),
|
||
}
|
||
return ok, message, {
|
||
"node_code": normalized_node_code,
|
||
"requested_by": normalized_requested_by,
|
||
"execution_mode": "control-plane",
|
||
"bootstrap_plan_request": bootstrap_plan_request,
|
||
"onboarding": onboarding,
|
||
"playbook_run": data,
|
||
"follow_up": follow_up,
|
||
}
|
||
|
||
|
||
def preview_managed_node_onboarding_acceptance(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
acceptance = dict(onboarding.get("acceptance") or {})
|
||
acceptance_status_code = str(acceptance.get("status_code") or "").strip()
|
||
if acceptance_status_code not in {"ready", "ssh_only"}:
|
||
return False, (
|
||
str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收"
|
||
), {"onboarding": onboarding}
|
||
|
||
from app.services.ops_playbook_service import preview_ops_playbook
|
||
|
||
normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip()
|
||
normalized_requested_by = str(requested_by or "api").strip() or "api"
|
||
execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
ok, message, data = preview_ops_playbook(
|
||
{
|
||
"playbook_key": "onboarding.acceptance",
|
||
"node_codes": [normalized_node_code] if normalized_node_code else [],
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": True,
|
||
"requested_by": normalized_requested_by,
|
||
}
|
||
)
|
||
follow_up = {
|
||
"next_stage_code": "ready",
|
||
"summary": (
|
||
str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip()
|
||
or "接管验收预览已生成；正式执行后应继续确认 health / node-agent / worker 三段验收全部通过。"
|
||
),
|
||
"recommended_commands": [
|
||
build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"),
|
||
] if normalized_node_code else [],
|
||
"focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})),
|
||
}
|
||
return ok, message, {
|
||
"node_code": normalized_node_code,
|
||
"requested_by": normalized_requested_by,
|
||
"execution_mode": execution_mode,
|
||
"onboarding": onboarding,
|
||
"acceptance": acceptance,
|
||
"playbook_preview": data,
|
||
"follow_up": follow_up,
|
||
}
|
||
|
||
|
||
def execute_managed_node_onboarding_acceptance(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
acceptance = dict(onboarding.get("acceptance") or {})
|
||
acceptance_status_code = str(acceptance.get("status_code") or "").strip()
|
||
if acceptance_status_code not in {"ready", "ssh_only"}:
|
||
return False, (
|
||
str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收"
|
||
), {"onboarding": onboarding}
|
||
|
||
from app.services.ops_playbook_service import execute_ops_playbook
|
||
|
||
normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip()
|
||
normalized_requested_by = str(requested_by or "api").strip() or "api"
|
||
execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": "onboarding.acceptance",
|
||
"node_codes": [normalized_node_code] if normalized_node_code else [],
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": True,
|
||
"requested_by": normalized_requested_by,
|
||
}
|
||
)
|
||
follow_up = {
|
||
"next_stage_code": "ready",
|
||
"summary": (
|
||
str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip()
|
||
or "接管验收已发起；接下来应继续确认 health / node-agent / worker 三段验收全部通过。"
|
||
),
|
||
"recommended_commands": [
|
||
build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code),
|
||
build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"),
|
||
] if normalized_node_code else [],
|
||
"focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})),
|
||
}
|
||
return ok, message, {
|
||
"node_code": normalized_node_code,
|
||
"requested_by": normalized_requested_by,
|
||
"execution_mode": execution_mode,
|
||
"onboarding": onboarding,
|
||
"acceptance": acceptance,
|
||
"playbook_run": data,
|
||
"follow_up": follow_up,
|
||
}
|
||
|
||
|
||
def preview_managed_node_onboarding_recovery(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
recovery = dict(onboarding.get("recovery_decision") or {})
|
||
recovery_action = str(recovery.get("action") or "").strip()
|
||
if recovery_action == "bootstrap_run":
|
||
ok, message, data = preview_managed_node_onboarding_bootstrap(
|
||
node_code=str(onboarding.get("node_code") or node_code or "").strip(),
|
||
requested_by=requested_by,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
return ok, message, {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
"selected_preview": data,
|
||
}
|
||
if recovery_action == "run_acceptance":
|
||
ok, message, data = preview_managed_node_onboarding_acceptance(
|
||
node_code=str(onboarding.get("node_code") or node_code or "").strip(),
|
||
requested_by=requested_by,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
return ok, message, {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
"selected_preview": data,
|
||
}
|
||
return False, (
|
||
str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作"
|
||
), {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
}
|
||
|
||
|
||
def execute_managed_node_onboarding_recovery(
|
||
*,
|
||
node_code: str,
|
||
requested_by: str = "api",
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
participation_payload: dict | None = None,
|
||
nodes_payload: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
onboarding = get_managed_node_onboarding(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if not onboarding:
|
||
return False, "节点不存在", {}
|
||
|
||
recovery = dict(onboarding.get("recovery_decision") or {})
|
||
recovery_action = str(recovery.get("action") or "").strip()
|
||
if recovery_action == "bootstrap_run":
|
||
ok, message, data = execute_managed_node_onboarding_bootstrap(
|
||
node_code=str(onboarding.get("node_code") or node_code or "").strip(),
|
||
requested_by=requested_by,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
return ok, message, {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
"selected_run": data,
|
||
}
|
||
if recovery_action == "run_acceptance":
|
||
ok, message, data = execute_managed_node_onboarding_acceptance(
|
||
node_code=str(onboarding.get("node_code") or node_code or "").strip(),
|
||
requested_by=requested_by,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
participation_payload=participation_payload,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
return ok, message, {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
"selected_run": data,
|
||
}
|
||
return False, (
|
||
str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作"
|
||
), {
|
||
"recovery_action": recovery_action,
|
||
"recovery_decision": recovery,
|
||
"onboarding": onboarding,
|
||
}
|
||
|
||
|
||
def build_managed_node_handover_bootstrap_plan(
|
||
*,
|
||
node_code: str,
|
||
issued_by: str = "api",
|
||
expires_in_hours: int = 72,
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "",
|
||
metadata: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
handover = get_managed_node_handover(
|
||
node_code,
|
||
control_plane_base_url=control_plane_base_url,
|
||
root_dir=root_dir,
|
||
)
|
||
if not handover:
|
||
return False, "节点不存在", {}
|
||
|
||
node = dict(handover.get("node") or {})
|
||
bootstrap_defaults = dict(handover.get("bootstrap_defaults") or {})
|
||
normalized_node_code = str(node.get("node_code") or node_code or "").strip()
|
||
ok, message, data = build_node_agent_bootstrap_plan(
|
||
node_code=normalized_node_code,
|
||
node_region=str(
|
||
bootstrap_defaults.get("node_region") or node.get("region") or "mainland"
|
||
).strip()
|
||
or "mainland",
|
||
node_role=str(bootstrap_defaults.get("node_role") or node.get("role") or "worker").strip() or "worker",
|
||
issued_by=str(issued_by or "api").strip() or "api",
|
||
expires_in_hours=int(expires_in_hours or bootstrap_defaults.get("expires_in_hours") or 72),
|
||
control_plane_base_url=str(
|
||
control_plane_base_url or bootstrap_defaults.get("control_plane_base_url") or handover.get("control_plane_base_url") or ""
|
||
).strip(),
|
||
root_dir=str(root_dir or bootstrap_defaults.get("root_dir") or handover.get("root_dir") or "/opt/domaincheck").strip()
|
||
or "/opt/domaincheck",
|
||
metadata={
|
||
**dict(bootstrap_defaults.get("metadata") or {}),
|
||
**dict(metadata or {}),
|
||
"issued_from": str((metadata or {}).get("issued_from") or "ops-managed-node-handover").strip()
|
||
or "ops-managed-node-handover",
|
||
},
|
||
)
|
||
if not ok:
|
||
return False, message, data
|
||
|
||
refreshed_handover = get_managed_node_handover(
|
||
normalized_node_code,
|
||
control_plane_base_url=str(handover.get("control_plane_base_url") or "").strip(),
|
||
root_dir=str(handover.get("root_dir") or "").strip(),
|
||
)
|
||
return True, message, {
|
||
**data,
|
||
"handover": refreshed_handover,
|
||
"requested_via": "managed-node-handover",
|
||
}
|
||
|
||
|
||
def _agent_job_type(job: dict) -> str:
|
||
normalized_job = dict(job or {})
|
||
metadata = dict(normalized_job.get("metadata") or {})
|
||
payload = dict(normalized_job.get("payload") or {})
|
||
explicit_job_type = str(metadata.get("job_type") or payload.get("job_type") or "").strip()
|
||
if explicit_job_type:
|
||
return explicit_job_type
|
||
action = str(normalized_job.get("action") or "").strip()
|
||
rollout_id = int(normalized_job.get("rollout_id") or 0)
|
||
if rollout_id > 0:
|
||
return "release_rollout"
|
||
if action == "deploy.release":
|
||
return "release_deploy"
|
||
if action in {"health.snapshot", "logs.collect", "diagnostics.collect"}:
|
||
return "inspection"
|
||
return "ops_action"
|
||
|
||
|
||
def _agent_release_context(job: dict) -> dict:
|
||
normalized_job = dict(job or {})
|
||
metadata = dict(normalized_job.get("metadata") or {})
|
||
payload = dict(normalized_job.get("payload") or {})
|
||
return {
|
||
"release_id": int(payload.get("release_id") or metadata.get("release_id") or 0),
|
||
"release_version": str(payload.get("release_version") or metadata.get("release_version") or "").strip(),
|
||
"rollout_id": int(
|
||
normalized_job.get("rollout_id")
|
||
or payload.get("rollout_id")
|
||
or metadata.get("rollout_id")
|
||
or 0
|
||
),
|
||
"rollout_code": str(payload.get("rollout_code") or metadata.get("rollout_code") or "").strip(),
|
||
}
|
||
|
||
|
||
def _agent_job_envelope(job: dict) -> dict:
|
||
normalized_job = dict(job or {})
|
||
steps = list(normalized_job.get("steps") or [])
|
||
first_step = dict(steps[0] or {}) if steps else {}
|
||
metadata = dict(normalized_job.get("metadata") or {})
|
||
focus_ref = dict(normalized_job.get("focus_ref") or {})
|
||
if not focus_ref:
|
||
focus_ref = {
|
||
"kind": "ops_job",
|
||
"job_id": int(normalized_job.get("id") or 0),
|
||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||
"action": str(normalized_job.get("action") or "").strip(),
|
||
"target_node_code": str(normalized_job.get("target_node_code") or "").strip(),
|
||
}
|
||
step_key = str(first_step.get("step_key") or metadata.get("step_key") or "dispatch").strip() or "dispatch"
|
||
step_title = (
|
||
str(
|
||
first_step.get("title")
|
||
or metadata.get("step_title")
|
||
or normalized_job.get("summary")
|
||
or normalized_job.get("action")
|
||
or ""
|
||
).strip()
|
||
or step_key
|
||
)
|
||
envelope = {
|
||
**normalized_job,
|
||
"protocol_version": "ops-agent/v1",
|
||
"envelope_type": "agent_job",
|
||
"job_id": int(normalized_job.get("id") or 0),
|
||
"job_type": _agent_job_type(normalized_job),
|
||
"step_key": step_key,
|
||
"step_title": step_title,
|
||
"policy": dict(normalized_job.get("policy") or {}),
|
||
"release_context": _agent_release_context(normalized_job),
|
||
"focus_ref": focus_ref,
|
||
"job_ref": {
|
||
"job_id": int(normalized_job.get("id") or 0),
|
||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||
"action": str(normalized_job.get("action") or "").strip(),
|
||
"target_node_code": str(normalized_job.get("target_node_code") or "").strip(),
|
||
},
|
||
"step_ref": {
|
||
"step_id": int(first_step.get("id") or 0),
|
||
"step_key": step_key,
|
||
"step_title": step_title,
|
||
},
|
||
}
|
||
return envelope
|
||
|
||
|
||
def _completion_summary(job: dict, *, client_request_id: str = "", deduplicated: bool = False) -> dict:
|
||
normalized_job = dict(job or {})
|
||
result = dict(normalized_job.get("result") or {})
|
||
return {
|
||
"job_id": int(normalized_job.get("id") or 0),
|
||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||
"status": str(normalized_job.get("status") or "").strip(),
|
||
"status_label": str(normalized_job.get("status_label") or "").strip(),
|
||
"deduplicated": bool(deduplicated),
|
||
"client_request_id": str(client_request_id or "").strip(),
|
||
"result_summary_text": str(result.get("summary_text") or result.get("summary") or "").strip(),
|
||
"focus_ref": dict(normalized_job.get("focus_ref") or {}),
|
||
}
|
||
|
||
|
||
def _slim_delivery_queue_head_record(node: dict, *, state: str) -> dict:
|
||
normalized_state = str(state or "").strip()
|
||
if normalized_state not in {"pending", "dead_letter"}:
|
||
return {}
|
||
|
||
if normalized_state == "pending":
|
||
created_at = str(node.get("delivery_queue_oldest_pending_at") or "").strip()
|
||
request_id = str(node.get("delivery_queue_oldest_pending_request_id") or "").strip()
|
||
request_kind = str(node.get("delivery_queue_oldest_pending_kind") or "").strip()
|
||
else:
|
||
created_at = str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip()
|
||
request_id = str(node.get("delivery_queue_oldest_dead_letter_request_id") or "").strip()
|
||
request_kind = str(node.get("delivery_queue_oldest_dead_letter_kind") or "").strip()
|
||
|
||
if not any((created_at, request_id, request_kind)):
|
||
return {}
|
||
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
return {
|
||
"record_key": f"{node_code}:{normalized_state}:{request_id or request_kind or created_at or 'head'}",
|
||
"record_id": request_id,
|
||
"node_code": node_code,
|
||
"state": normalized_state,
|
||
"label": "待重试头部记录" if normalized_state == "pending" else "死信头部记录",
|
||
"request_kind": request_kind,
|
||
"client_request_id": request_id,
|
||
"created_at": created_at if normalized_state == "pending" else "",
|
||
"dead_letter_at": created_at if normalized_state == "dead_letter" else "",
|
||
"detail_code": "",
|
||
"error_message": "",
|
||
"record_source": "agent-heartbeat-snapshot",
|
||
"record_visibility": "head_only",
|
||
"payload_visibility": "unavailable",
|
||
"response_visibility": "unavailable",
|
||
}
|
||
|
||
|
||
def get_managed_node_delivery_queue(node_code: str) -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return {}
|
||
|
||
payload = list_managed_nodes_with_agent_state()
|
||
node = next(
|
||
(
|
||
dict(item or {})
|
||
for item in list(payload.get("nodes") or [])
|
||
if str(item.get("node_code") or "").strip() == normalized_node_code
|
||
),
|
||
{},
|
||
)
|
||
if not node:
|
||
return {}
|
||
|
||
pending_head = _slim_delivery_queue_head_record(node, state="pending")
|
||
dead_letter_head = _slim_delivery_queue_head_record(node, state="dead_letter")
|
||
record_visibility = "head_only"
|
||
capabilities = {
|
||
"can_flush": True,
|
||
"can_replay": True,
|
||
"can_discard": True,
|
||
}
|
||
|
||
return {
|
||
"node_code": normalized_node_code,
|
||
"node": {
|
||
"node_code": normalized_node_code,
|
||
"title": str(node.get("title") or "").strip(),
|
||
"region": str(node.get("region") or "").strip(),
|
||
"role": str(node.get("role") or "").strip(),
|
||
"is_managed": bool(node.get("is_managed", False)),
|
||
"is_enabled": bool(node.get("is_enabled", False)),
|
||
"is_agent_online": bool(node.get("is_agent_online", False)),
|
||
"agent_state": str(node.get("agent_state") or "").strip(),
|
||
"agent_state_label": str(node.get("agent_state_label") or "").strip(),
|
||
"remote_access_state": str(node.get("remote_access_state") or "").strip(),
|
||
"remote_access_label": str(node.get("remote_access_label") or "").strip(),
|
||
},
|
||
"summary": {
|
||
"state": str(node.get("delivery_queue_state") or "").strip(),
|
||
"label": str(node.get("delivery_queue_label") or "").strip(),
|
||
"reason": str(node.get("delivery_queue_reason") or "").strip(),
|
||
"pending_count": int(node.get("delivery_queue_pending_count", 0) or 0),
|
||
"dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0),
|
||
"last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(),
|
||
"oldest_pending_at": str(node.get("delivery_queue_oldest_pending_at") or "").strip(),
|
||
"oldest_dead_letter_at": str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip(),
|
||
},
|
||
"record_visibility": record_visibility,
|
||
"capabilities": capabilities,
|
||
"records_capability": {
|
||
"record_source": "agent-heartbeat-snapshot",
|
||
"record_visibility": record_visibility,
|
||
"full_records_available": False,
|
||
"available_actions": [
|
||
{
|
||
"key": "delivery.queue.flush",
|
||
"label": "立即冲刷",
|
||
"endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/flush",
|
||
"record_scope": "pending",
|
||
"execution_mode": "remote-agent",
|
||
},
|
||
{
|
||
"key": "delivery.queue.replay",
|
||
"label": "重放死信",
|
||
"endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/replay",
|
||
"record_scope": "dead_letter",
|
||
"execution_mode": "remote-agent",
|
||
},
|
||
{
|
||
"key": "delivery.queue.discard",
|
||
"label": "丢弃死信",
|
||
"endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/records/{{record_id}}/discard",
|
||
"record_scope": "dead_letter",
|
||
"execution_mode": "remote-agent",
|
||
},
|
||
],
|
||
"next_step": "当前控制面仍以头部记录视角展示队列；已支持通过正式 Ops Job 触发冲刷、重放和丢弃动作。",
|
||
},
|
||
"head_records": {
|
||
"pending": pending_head,
|
||
"dead_letter": dead_letter_head,
|
||
},
|
||
}
|
||
|
||
|
||
def list_managed_node_delivery_queue_records(
|
||
node_code: str,
|
||
*,
|
||
state: str = "",
|
||
limit: int = 50,
|
||
) -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
normalized_state = str(state or "").strip()
|
||
safe_limit = min(max(int(limit or 50), 1), 200)
|
||
queue_payload = get_managed_node_delivery_queue(normalized_node_code)
|
||
if not queue_payload:
|
||
return {}
|
||
|
||
candidate_records = [
|
||
record
|
||
for record in (
|
||
dict((queue_payload.get("head_records") or {}).get("pending") or {}),
|
||
dict((queue_payload.get("head_records") or {}).get("dead_letter") or {}),
|
||
)
|
||
if record
|
||
]
|
||
if normalized_state:
|
||
candidate_records = [
|
||
record for record in candidate_records if str(record.get("state") or "").strip() == normalized_state
|
||
]
|
||
records = candidate_records[:safe_limit]
|
||
|
||
available_state_counts: dict[str, int] = {}
|
||
for record in candidate_records:
|
||
record_state = str(record.get("state") or "").strip() or "unknown"
|
||
available_state_counts[record_state] = int(available_state_counts.get(record_state, 0) or 0) + 1
|
||
|
||
return {
|
||
"node_code": normalized_node_code,
|
||
"records": records,
|
||
"summary": {
|
||
"total": len(records),
|
||
"returned_records": len(records),
|
||
"available_total": len(candidate_records),
|
||
"available_state_counts": available_state_counts,
|
||
"record_visibility": "head_only",
|
||
"filters": {
|
||
"state": normalized_state,
|
||
"limit": safe_limit,
|
||
},
|
||
},
|
||
"records_capability": dict(queue_payload.get("records_capability") or {}),
|
||
"record_visibility": str((queue_payload.get("record_visibility") or "head_only")).strip() or "head_only",
|
||
"capabilities": dict(queue_payload.get("capabilities") or {}),
|
||
}
|
||
|
||
|
||
def _delivery_queue_visible_record_id(record: dict | None = None) -> str:
|
||
normalized_record = dict(record or {})
|
||
return (
|
||
str(normalized_record.get("record_id") or "").strip()
|
||
or str(normalized_record.get("client_request_id") or "").strip()
|
||
or str(normalized_record.get("record_key") or "").strip()
|
||
)
|
||
|
||
|
||
def _find_visible_delivery_queue_record(queue_payload: dict, record_id: str) -> dict:
|
||
normalized_record_id = str(record_id or "").strip()
|
||
if not normalized_record_id:
|
||
return {}
|
||
for state in ("pending", "dead_letter"):
|
||
record = dict(((queue_payload.get("head_records") or {}).get(state) or {}))
|
||
if record and _delivery_queue_visible_record_id(record) == normalized_record_id:
|
||
return record
|
||
return {}
|
||
|
||
|
||
def _build_delivery_queue_selector_from_template_payload(
|
||
template_key: str,
|
||
normalized_payload: dict,
|
||
*,
|
||
force_state: str = "",
|
||
) -> dict:
|
||
selector: dict[str, object] = {}
|
||
if str(force_state or "").strip():
|
||
selector["state"] = str(force_state or "").strip()
|
||
elif template_key in {"delivery.queue.replay", "delivery.queue.discard"}:
|
||
selector["state"] = "dead_letter"
|
||
|
||
record_id = str(normalized_payload.get("record_id") or "").strip()
|
||
request_kind = str(normalized_payload.get("request_kind") or "").strip()
|
||
detail_code = str(normalized_payload.get("detail_code") or "").strip()
|
||
if record_id:
|
||
selector["record_id"] = record_id
|
||
if request_kind:
|
||
selector["request_kind"] = request_kind
|
||
if detail_code:
|
||
selector["detail_code"] = detail_code
|
||
return selector
|
||
|
||
|
||
def request_managed_node_delivery_queue_action(
|
||
node_code: str,
|
||
action_key: str,
|
||
*,
|
||
payload: dict | None = None,
|
||
record_id: str = "",
|
||
) -> tuple[bool, str, dict]:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
normalized_action_key = str(action_key or "").strip()
|
||
raw_payload = dict(payload or {})
|
||
if not normalized_node_code:
|
||
return False, "node_code 不能为空", {}
|
||
if normalized_action_key not in {"delivery.queue.flush", "delivery.queue.replay", "delivery.queue.discard"}:
|
||
return False, "当前不支持该 delivery queue 动作", {"action_key": normalized_action_key}
|
||
|
||
queue_payload = get_managed_node_delivery_queue(normalized_node_code)
|
||
if not queue_payload:
|
||
return False, "节点不存在", {}
|
||
|
||
template = get_ops_action_template(normalized_action_key)
|
||
if not template:
|
||
return False, "delivery queue 动作模板不存在", {"action_key": normalized_action_key}
|
||
|
||
effective_payload = dict(raw_payload)
|
||
if str(record_id or "").strip():
|
||
effective_payload["record_id"] = str(record_id or "").strip()
|
||
normalize_ok, normalize_message, normalized_action_payload = build_ops_template_payload(
|
||
normalized_action_key,
|
||
effective_payload,
|
||
)
|
||
if not normalize_ok:
|
||
return False, normalize_message, {"action_key": normalized_action_key}
|
||
|
||
visible_record = {}
|
||
if str(record_id or "").strip():
|
||
visible_record = _find_visible_delivery_queue_record(queue_payload, str(record_id or "").strip())
|
||
if not visible_record:
|
||
return False, "当前控制面仅支持对可见头部记录执行单条动作", {
|
||
"action_key": normalized_action_key,
|
||
"node_code": normalized_node_code,
|
||
"record_id": str(record_id or "").strip(),
|
||
"record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")),
|
||
}
|
||
if normalized_action_key in {"delivery.queue.replay", "delivery.queue.discard"}:
|
||
if str(visible_record.get("state") or "").strip() != "dead_letter":
|
||
return False, "单条重放/丢弃当前仅支持死信头部记录", {
|
||
"action_key": normalized_action_key,
|
||
"node_code": normalized_node_code,
|
||
"record": visible_record,
|
||
}
|
||
normalized_action_payload["record_id"] = _delivery_queue_visible_record_id(visible_record)
|
||
|
||
selector = _build_delivery_queue_selector_from_template_payload(
|
||
normalized_action_key,
|
||
normalized_action_payload,
|
||
)
|
||
action_payload: dict[str, object] = {}
|
||
if normalized_action_key == "delivery.queue.flush":
|
||
action_payload["limit"] = int(normalized_action_payload.get("limit") or 20)
|
||
elif normalized_action_key == "delivery.queue.replay":
|
||
action_payload["selector"] = selector
|
||
action_payload["limit"] = int(normalized_action_payload.get("limit") or 20)
|
||
action_payload["flush_after_replay"] = bool(normalized_action_payload.get("flush_after_replay", True))
|
||
if str(normalized_action_payload.get("reason") or "").strip():
|
||
action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip()
|
||
elif normalized_action_key == "delivery.queue.discard":
|
||
action_payload["selector"] = selector
|
||
action_payload["limit"] = int(normalized_action_payload.get("limit") or 20)
|
||
action_payload["discarded_by"] = (
|
||
str(raw_payload.get("discarded_by") or raw_payload.get("requested_by") or "web-ui").strip() or "web-ui"
|
||
)
|
||
action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip()
|
||
|
||
requested_by = str(raw_payload.get("requested_by") or "web-ui").strip() or "web-ui"
|
||
ok, message, data = create_ops_job(
|
||
{
|
||
"action": normalized_action_key,
|
||
"target_type": "node",
|
||
"target_node_code": normalized_node_code,
|
||
"requested_by": requested_by,
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": bool(template.get("default_auto_approve", False)),
|
||
"payload": action_payload,
|
||
"metadata": {
|
||
"delivery_queue_action": {
|
||
"action_key": normalized_action_key,
|
||
"template_key": normalized_action_key,
|
||
"selector": selector,
|
||
"record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")),
|
||
"requested_from": "ops-delivery-queue",
|
||
"visible_record": visible_record,
|
||
"queue_summary": dict(queue_payload.get("summary") or {}),
|
||
}
|
||
},
|
||
}
|
||
)
|
||
if not ok:
|
||
return False, message, data
|
||
|
||
action_label = str(template.get("title") or normalized_action_key).strip()
|
||
return True, f"{action_label}任务已创建", {
|
||
"node_code": normalized_node_code,
|
||
"action_key": normalized_action_key,
|
||
"action_label": action_label,
|
||
"queue": {
|
||
"summary": dict(queue_payload.get("summary") or {}),
|
||
"records_capability": dict(queue_payload.get("records_capability") or {}),
|
||
},
|
||
"selector": selector,
|
||
"visible_record": visible_record,
|
||
"job": dict(data.get("job") or {}),
|
||
"executed_immediately": bool(data.get("executed_immediately", False)),
|
||
}
|
||
|
||
|
||
def issue_node_agent_token(
|
||
*,
|
||
node_code: str,
|
||
issued_by: str = "api",
|
||
expires_in_hours: int = 72,
|
||
metadata: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
ensure_ops_agent_schema()
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return False, "node_code 不能为空", {}
|
||
|
||
token = secrets.token_urlsafe(32)
|
||
token_hash = _hash_token(token)
|
||
expires_at = datetime.now() + timedelta(hours=max(1, min(int(expires_in_hours or 72), 24 * 30)))
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO ops_node_tokens (
|
||
token_hash, node_code, purpose, issued_by, is_enabled, expires_at, metadata_json, created_at, updated_at
|
||
) VALUES (%s, %s, 'agent', %s, TRUE, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id, expires_at, created_at
|
||
""",
|
||
(
|
||
token_hash,
|
||
normalized_node_code,
|
||
str(issued_by or "api").strip() or "api",
|
||
expires_at,
|
||
json.dumps(metadata or {}, ensure_ascii=False),
|
||
),
|
||
)
|
||
row = cur.fetchone()
|
||
conn.commit()
|
||
|
||
return True, "节点 Agent 令牌已签发", {
|
||
"token": token,
|
||
"token_preview": f"{token[:6]}...{token[-4:]}",
|
||
"record_id": int(row[0]),
|
||
"node_code": normalized_node_code,
|
||
"expires_at": _format_time(row[1]),
|
||
"created_at": _format_time(row[2]),
|
||
}
|
||
|
||
|
||
def _normalize_control_plane_base_url(raw_value: str) -> str:
|
||
normalized = str(raw_value or "").strip().rstrip("/")
|
||
if normalized.endswith("/api/v1"):
|
||
normalized = normalized[: -len("/api/v1")]
|
||
return normalized
|
||
|
||
|
||
def _candidate_node_agent_install_paths(root_dir: str) -> list[str]:
|
||
normalized_root_dir = str(root_dir or "/opt/domaincheck").strip().rstrip("/") or "/opt/domaincheck"
|
||
return [
|
||
f"{normalized_root_dir}/domainCheck/deploy/multi-region/bootstrap_node_agent.sh",
|
||
f"{normalized_root_dir}/domain-api/deploy/multi-region/bootstrap_node_agent.sh",
|
||
f"{normalized_root_dir}/deploy/multi-region/bootstrap_node_agent.sh",
|
||
]
|
||
|
||
|
||
def _build_node_agent_install_block(root_dir: str) -> tuple[str, str]:
|
||
candidates = _candidate_node_agent_install_paths(root_dir)
|
||
preferred_path = candidates[0]
|
||
lines = [
|
||
f"ROOT_DIR={shlex.quote(root_dir)}",
|
||
"INSTALL_SCRIPT=",
|
||
"for candidate in \\",
|
||
]
|
||
for index, candidate in enumerate(candidates):
|
||
suffix = " \\" if index < len(candidates) - 1 else ""
|
||
lines.append(f" {shlex.quote(candidate)}{suffix}")
|
||
lines.extend(
|
||
[
|
||
"do",
|
||
" if [[ -f \"${candidate}\" ]]; then",
|
||
" INSTALL_SCRIPT=\"${candidate}\"",
|
||
" break",
|
||
" fi",
|
||
"done",
|
||
"",
|
||
"if [[ -z \"${INSTALL_SCRIPT}\" ]]; then",
|
||
" echo \"bootstrap_node_agent.sh not found under known layouts\" >&2",
|
||
" exit 1",
|
||
"fi",
|
||
"",
|
||
"bash \"${INSTALL_SCRIPT}\" \"${ROOT_DIR}\"",
|
||
]
|
||
)
|
||
return preferred_path, "\n".join(lines)
|
||
|
||
|
||
def _build_node_agent_bootstrap_script(
|
||
*,
|
||
node_code: str,
|
||
root_dir: str,
|
||
install_script_path: str,
|
||
install_command_block: str,
|
||
env_content: str,
|
||
health_checks: list[str],
|
||
) -> tuple[str, str, str]:
|
||
script_name = f"bootstrap-node-agent-{node_code}.sh"
|
||
script_path = f"/tmp/{script_name}"
|
||
script_lines = [
|
||
"#!/usr/bin/env bash",
|
||
"set -euo pipefail",
|
||
"",
|
||
f"ROOT_DIR={shlex.quote(root_dir)}",
|
||
f"INSTALL_SCRIPT={shlex.quote(install_script_path)}",
|
||
"ENV_FILE=/etc/default/domaincheck-node-agent",
|
||
"SERVICE_NAME=domaincheck-node-agent",
|
||
"",
|
||
"# Resolve bootstrap_node_agent.sh across supported repo layouts.",
|
||
*install_command_block.splitlines(),
|
||
"",
|
||
"cat >\"${ENV_FILE}\" <<'EOF'",
|
||
env_content,
|
||
"EOF",
|
||
"",
|
||
"systemctl restart \"${SERVICE_NAME}\"",
|
||
"systemctl status \"${SERVICE_NAME}\" --no-pager -l",
|
||
"journalctl -u \"${SERVICE_NAME}\" -n 60 --no-pager",
|
||
]
|
||
if health_checks:
|
||
script_lines.extend(
|
||
[
|
||
"",
|
||
"echo",
|
||
"echo '[health-checks]'",
|
||
]
|
||
)
|
||
for item in health_checks:
|
||
normalized_item = str(item or "").strip()
|
||
if normalized_item:
|
||
script_lines.append(normalized_item)
|
||
script_content = "\n".join(script_lines).strip() + "\n"
|
||
write_script_block = "\n".join(
|
||
[
|
||
f"cat >{script_path} <<'EOF'",
|
||
script_content.rstrip("\n"),
|
||
"EOF",
|
||
f"chmod +x {script_path}",
|
||
]
|
||
)
|
||
run_script_block = "\n".join(
|
||
[
|
||
write_script_block,
|
||
f"bash {script_path}",
|
||
]
|
||
)
|
||
return script_name, script_content, run_script_block
|
||
|
||
|
||
def build_node_agent_bootstrap_plan(
|
||
*,
|
||
node_code: str,
|
||
node_region: str = "mainland",
|
||
node_role: str = "worker",
|
||
issued_by: str = "api",
|
||
expires_in_hours: int = 72,
|
||
control_plane_base_url: str = "",
|
||
root_dir: str = "/opt/domaincheck",
|
||
metadata: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return False, "node_code 不能为空", {}
|
||
|
||
normalized_region = str(node_region or "mainland").strip() or "mainland"
|
||
normalized_role = str(node_role or "worker").strip() or "worker"
|
||
normalized_root_dir = str(root_dir or "/opt/domaincheck").strip() or "/opt/domaincheck"
|
||
normalized_control_plane_base_url = _normalize_control_plane_base_url(control_plane_base_url)
|
||
if not normalized_control_plane_base_url:
|
||
normalized_control_plane_base_url = f"http://127.0.0.1:{settings.api_port}"
|
||
|
||
token_ok, token_message, token_data = issue_node_agent_token(
|
||
node_code=normalized_node_code,
|
||
issued_by=issued_by,
|
||
expires_in_hours=expires_in_hours,
|
||
metadata=metadata or {},
|
||
)
|
||
if not token_ok:
|
||
return False, token_message, {}
|
||
|
||
env_lines = [
|
||
f"OPS_CONTROL_PLANE_BASE_URL={normalized_control_plane_base_url}",
|
||
f"OPS_AGENT_TOKEN={token_data.get('token') or ''}",
|
||
"OPS_AGENT_POLL_INTERVAL_SECONDS=5",
|
||
"",
|
||
f"NODE_CODE={normalized_node_code}",
|
||
f"NODE_REGION={normalized_region}",
|
||
f"NODE_ROLE={normalized_role}",
|
||
"",
|
||
f"WORKER_SERVICE_NAME={settings.worker_service_name}",
|
||
f"API_SERVICE_NAME={settings.api_service_name}",
|
||
f"SYNC_AGENT_SERVICE_NAME={settings.sync_agent_service_name}",
|
||
"NODE_AGENT_SERVICE_NAME=domaincheck-node-agent",
|
||
"",
|
||
'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]',
|
||
"OPS_AGENT_LABELS={}",
|
||
]
|
||
env_content = "\n".join(env_lines)
|
||
install_script_path, install_command_block = _build_node_agent_install_block(normalized_root_dir)
|
||
command_lines = [
|
||
install_command_block,
|
||
"cat >/etc/default/domaincheck-node-agent <<'EOF'",
|
||
env_content,
|
||
"EOF",
|
||
"systemctl restart domaincheck-node-agent",
|
||
"systemctl status domaincheck-node-agent --no-pager -l",
|
||
"journalctl -u domaincheck-node-agent -n 60 --no-pager",
|
||
]
|
||
command_block = "\n".join(command_lines)
|
||
health_checks = [
|
||
"systemctl status domaincheck-node-agent --no-pager -l",
|
||
"journalctl -u domaincheck-node-agent -n 60 --no-pager",
|
||
f"curl -s {normalized_control_plane_base_url}/api/v1/ops/overview",
|
||
]
|
||
bootstrap_script_name, bootstrap_script_content, bootstrap_run_script_block = _build_node_agent_bootstrap_script(
|
||
node_code=normalized_node_code,
|
||
root_dir=normalized_root_dir,
|
||
install_script_path=install_script_path,
|
||
install_command_block=install_command_block,
|
||
env_content=env_content,
|
||
health_checks=health_checks,
|
||
)
|
||
|
||
return True, "节点 Agent 接入方案已生成", {
|
||
**token_data,
|
||
"control_plane_base_url": normalized_control_plane_base_url,
|
||
"bootstrap_plan": {
|
||
"node_code": normalized_node_code,
|
||
"node_region": normalized_region,
|
||
"node_role": normalized_role,
|
||
"root_dir": normalized_root_dir,
|
||
"env_file": "/etc/default/domaincheck-node-agent",
|
||
"service_name": "domaincheck-node-agent",
|
||
"service_file": "/etc/systemd/system/domaincheck-node-agent.service",
|
||
"install_script_path": install_script_path,
|
||
"install_script_candidates": _candidate_node_agent_install_paths(normalized_root_dir),
|
||
"install_command_block": install_command_block,
|
||
"env_content": env_content,
|
||
"command_lines": command_lines,
|
||
"command_block": command_block,
|
||
"health_checks": health_checks,
|
||
"health_check_block": "\n".join(health_checks),
|
||
"bootstrap_script_name": bootstrap_script_name,
|
||
"bootstrap_script_path": f"/tmp/{bootstrap_script_name}",
|
||
"bootstrap_script_content": bootstrap_script_content,
|
||
"bootstrap_run_script_block": bootstrap_run_script_block,
|
||
},
|
||
}
|
||
|
||
|
||
def _authenticate_agent_token(token: str, *, expected_node_code: str | None = None) -> tuple[bool, str, dict]:
|
||
ensure_ops_agent_schema()
|
||
token_hash = _hash_token(token)
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, node_code, purpose, is_enabled, expires_at, metadata_json
|
||
FROM ops_node_tokens
|
||
WHERE token_hash = %s
|
||
LIMIT 1
|
||
""",
|
||
(token_hash,),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return _agent_error("agent token 无效", "agent_token_invalid")
|
||
record_id = int(row[0])
|
||
node_code = str(row[1] or "")
|
||
purpose = str(row[2] or "")
|
||
is_enabled = bool(row[3])
|
||
expires_at = row[4]
|
||
metadata = _decode_json(row[5])
|
||
if purpose != "agent" or not is_enabled:
|
||
return _agent_error("agent token 不可用", "agent_token_unavailable")
|
||
if isinstance(expires_at, datetime):
|
||
current_time = datetime.now(expires_at.tzinfo) if expires_at.tzinfo else datetime.now()
|
||
if expires_at < current_time:
|
||
return _agent_error("agent token 已过期", "agent_token_expired")
|
||
normalized_expected = str(expected_node_code or "").strip()
|
||
if normalized_expected and normalized_expected != node_code:
|
||
return _agent_error(
|
||
"agent token 与节点不匹配",
|
||
"agent_token_node_mismatch",
|
||
{"expected_node_code": normalized_expected, "token_node_code": node_code},
|
||
)
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_node_tokens
|
||
SET last_used_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(record_id,),
|
||
)
|
||
conn.commit()
|
||
return True, "ok", {
|
||
"record_id": record_id,
|
||
"node_code": node_code,
|
||
"metadata": metadata,
|
||
"expires_at": _format_time(expires_at),
|
||
}
|
||
|
||
|
||
def _upsert_agent_runtime(node_code: str, payload: dict) -> None:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return
|
||
|
||
region = str(payload.get("region") or "unknown").strip() or "unknown"
|
||
role = str(payload.get("role") or "worker").strip() or "worker"
|
||
title = str(payload.get("title") or normalized_node_code).strip() or normalized_node_code
|
||
metadata = payload.get("metadata") or {}
|
||
ssh_host = str(payload.get("ssh_host") or "").strip()
|
||
ssh_user = str(payload.get("ssh_user") or "").strip()
|
||
ssh_port = max(1, int(payload.get("ssh_port") or 22))
|
||
auth_mode = str(payload.get("auth_mode") or "key").strip() or "key"
|
||
deploy_channel = str(payload.get("deploy_channel") or "stable").strip() or "stable"
|
||
|
||
merged_metadata = {
|
||
**metadata,
|
||
"agent_version": str(payload.get("agent_version") or metadata.get("agent_version") or "").strip(),
|
||
"last_seen_at": datetime.now().isoformat(timespec="seconds"),
|
||
"capabilities": payload.get("capabilities") or metadata.get("capabilities") or [],
|
||
"labels": payload.get("labels") or metadata.get("labels") or {},
|
||
"hostname": str(payload.get("hostname") or metadata.get("hostname") or "").strip(),
|
||
"ip": str(payload.get("ip") or metadata.get("ip") or "").strip(),
|
||
}
|
||
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO ops_managed_nodes (
|
||
node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, TRUE, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
ON CONFLICT (node_code) DO UPDATE SET
|
||
region = EXCLUDED.region,
|
||
role = EXCLUDED.role,
|
||
title = EXCLUDED.title,
|
||
ssh_host = CASE WHEN EXCLUDED.ssh_host <> '' THEN EXCLUDED.ssh_host ELSE ops_managed_nodes.ssh_host END,
|
||
ssh_port = EXCLUDED.ssh_port,
|
||
ssh_user = CASE WHEN EXCLUDED.ssh_user <> '' THEN EXCLUDED.ssh_user ELSE ops_managed_nodes.ssh_user END,
|
||
auth_mode = EXCLUDED.auth_mode,
|
||
deploy_channel = EXCLUDED.deploy_channel,
|
||
metadata_json = EXCLUDED.metadata_json,
|
||
last_seen_at = CURRENT_TIMESTAMP,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
""",
|
||
(
|
||
normalized_node_code,
|
||
region,
|
||
role,
|
||
title,
|
||
ssh_host,
|
||
ssh_port,
|
||
ssh_user,
|
||
auth_mode,
|
||
deploy_channel,
|
||
json.dumps(merged_metadata, ensure_ascii=False),
|
||
),
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, auth
|
||
_upsert_agent_runtime(node_code, payload)
|
||
return True, "Agent 注册成功", {
|
||
"node_code": node_code,
|
||
"expires_at": auth.get("expires_at", ""),
|
||
"capabilities": payload.get("capabilities") or [],
|
||
}
|
||
|
||
|
||
def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, auth
|
||
_upsert_agent_runtime(node_code, payload)
|
||
return True, "heartbeat ok", {
|
||
"node_code": node_code,
|
||
"server_time": _format_time(datetime.now()),
|
||
"expires_at": auth.get("expires_at", ""),
|
||
}
|
||
|
||
|
||
def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, _auth
|
||
|
||
safe_limit = min(max(int(limit or 1), 1), 10)
|
||
with get_db() as conn:
|
||
conn.autocommit = False
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM ops_jobs
|
||
WHERE target_node_code = %s
|
||
AND status = 'queued'
|
||
AND execution_mode = 'remote-agent'
|
||
ORDER BY created_at ASC, id ASC
|
||
LIMIT %s
|
||
FOR UPDATE SKIP LOCKED
|
||
""",
|
||
(node_code, safe_limit),
|
||
)
|
||
rows = cur.fetchall()
|
||
job_ids = [int(row[0]) for row in rows]
|
||
jobs: list[dict] = []
|
||
for job_id in job_ids:
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_jobs
|
||
SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), dispatched_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(job_id,),
|
||
)
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_job_steps
|
||
SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP
|
||
WHERE job_id = %s
|
||
""",
|
||
(job_id,),
|
||
)
|
||
append_ops_job_event(
|
||
job_id=job_id,
|
||
node_code=node_code,
|
||
event_type="agent_dispatched",
|
||
message=f"任务已派发给节点 {node_code}",
|
||
payload={"node_code": node_code},
|
||
)
|
||
jobs.append(_agent_job_envelope(get_ops_job(job_id)))
|
||
conn.commit()
|
||
return True, "ok", {
|
||
"jobs": jobs,
|
||
"count": len(jobs),
|
||
"node_code": node_code,
|
||
"limit": safe_limit,
|
||
"protocol_version": "ops-agent/v1",
|
||
"envelope_type": "agent_job",
|
||
}
|
||
|
||
|
||
def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, _auth
|
||
|
||
with get_db() as conn:
|
||
conn.autocommit = False
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_jobs
|
||
SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s AND target_node_code = %s
|
||
RETURNING id
|
||
""",
|
||
(int(job_id), node_code),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
conn.rollback()
|
||
return _agent_error(
|
||
"任务不存在或不属于当前节点",
|
||
"ops_job_not_owned_by_agent",
|
||
{"job_id": int(job_id), "node_code": node_code},
|
||
)
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_job_steps
|
||
SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP
|
||
WHERE job_id = %s
|
||
RETURNING id
|
||
""",
|
||
(int(job_id),),
|
||
)
|
||
step_rows = cur.fetchall()
|
||
step_ids = [int(item[0]) for item in step_rows]
|
||
append_ops_job_event(
|
||
job_id=int(job_id),
|
||
node_code=node_code,
|
||
event_type="agent_started",
|
||
message=f"节点 {node_code} 已开始执行任务",
|
||
payload={"step_ids": step_ids},
|
||
)
|
||
conn.commit()
|
||
job = get_ops_job(int(job_id))
|
||
return True, "任务已标记为运行中", {
|
||
"job": job,
|
||
"accepted_job": _agent_job_envelope(job),
|
||
"focus_ref": dict(job.get("focus_ref") or {}),
|
||
}
|
||
|
||
|
||
def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, _auth
|
||
|
||
job_status = str(payload.get("status") or "").strip().lower()
|
||
if job_status not in {"success", "failed", "partially_succeeded"}:
|
||
return _agent_error(
|
||
"status 必须是 success / failed / partially_succeeded",
|
||
"ops_job_invalid_status",
|
||
{"allowed_statuses": ["success", "failed", "partially_succeeded"]},
|
||
)
|
||
|
||
stdout_text = str(payload.get("stdout") or "")
|
||
stderr_text = str(payload.get("stderr") or "")
|
||
result = dict(payload.get("result") or {})
|
||
error_message = str(payload.get("error_message") or "").strip()
|
||
client_request_id = str(payload.get("client_request_id") or "").strip()[:128]
|
||
duration_ms = max(0, int(payload.get("duration_ms") or 0))
|
||
summary_text = str(payload.get("summary_text") or "").strip()
|
||
focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {}
|
||
if duration_ms and "duration_ms" not in result:
|
||
result["duration_ms"] = duration_ms
|
||
if summary_text and "summary_text" not in result and "summary" not in result:
|
||
result["summary_text"] = summary_text
|
||
if focus_ref and "focus_ref" not in result:
|
||
result["focus_ref"] = focus_ref
|
||
event_level = "info" if job_status == "success" else ("warning" if job_status == "partially_succeeded" else "error")
|
||
|
||
with get_db() as conn:
|
||
conn.autocommit = False
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, COALESCE(last_agent_complete_request_id, '')
|
||
FROM ops_jobs
|
||
WHERE id = %s AND target_node_code = %s
|
||
LIMIT 1
|
||
""",
|
||
(int(job_id), node_code),
|
||
)
|
||
job_row = cur.fetchone()
|
||
if not job_row:
|
||
conn.rollback()
|
||
return _agent_error(
|
||
"任务不存在或不属于当前节点",
|
||
"ops_job_not_owned_by_agent",
|
||
{"job_id": int(job_id), "node_code": node_code},
|
||
)
|
||
current_request_id = str(job_row[1] or "")
|
||
if client_request_id and current_request_id == client_request_id:
|
||
conn.rollback()
|
||
deduplicated_job = get_ops_job(int(job_id))
|
||
return True, "任务结果已幂等回写", {
|
||
"job": deduplicated_job,
|
||
"deduplicated": True,
|
||
"client_request_id": client_request_id,
|
||
"completion_summary": _completion_summary(
|
||
deduplicated_job,
|
||
client_request_id=client_request_id,
|
||
deduplicated=True,
|
||
),
|
||
}
|
||
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_jobs
|
||
SET
|
||
status = %s,
|
||
result_json = %s,
|
||
error_message = %s,
|
||
last_agent_complete_request_id = CASE WHEN %s <> '' THEN %s ELSE last_agent_complete_request_id END,
|
||
finished_at = CURRENT_TIMESTAMP,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(
|
||
job_status,
|
||
json.dumps(result, ensure_ascii=False),
|
||
error_message if job_status != "success" else "",
|
||
client_request_id,
|
||
client_request_id,
|
||
int(job_id),
|
||
),
|
||
)
|
||
cur.execute(
|
||
"""
|
||
UPDATE ops_job_steps
|
||
SET
|
||
status = %s,
|
||
stdout_text = %s,
|
||
stderr_text = %s,
|
||
result_json = %s,
|
||
finished_at = CURRENT_TIMESTAMP,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
WHERE job_id = %s
|
||
RETURNING id
|
||
""",
|
||
(
|
||
job_status,
|
||
stdout_text,
|
||
stderr_text,
|
||
json.dumps(result, ensure_ascii=False),
|
||
int(job_id),
|
||
),
|
||
)
|
||
step_rows = cur.fetchall()
|
||
step_ids = [int(item[0]) for item in step_rows]
|
||
append_ops_job_event(
|
||
job_id=int(job_id),
|
||
node_code=node_code,
|
||
client_event_id=(f"complete:{client_request_id}" if client_request_id else ""),
|
||
event_type="agent_completed",
|
||
message=f"节点 {node_code} 已完成任务，状态: {job_status}",
|
||
level=event_level,
|
||
payload={
|
||
"step_ids": step_ids,
|
||
"result": result,
|
||
"duration_ms": duration_ms,
|
||
"summary_text": str(result.get("summary_text") or result.get("summary") or summary_text).strip(),
|
||
"focus_ref": focus_ref,
|
||
},
|
||
)
|
||
conn.commit()
|
||
from app.services.ops_release_service import refresh_release_rollout_for_job
|
||
|
||
refresh_release_rollout_for_job(int(job_id))
|
||
job = get_ops_job(int(job_id))
|
||
return True, "任务结果已回写", {
|
||
"job": job,
|
||
"deduplicated": False,
|
||
"client_request_id": client_request_id,
|
||
"completion_summary": _completion_summary(job, client_request_id=client_request_id),
|
||
}
|
||
|
||
|
||
def agent_append_job_event(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]:
|
||
node_code = str(payload.get("node_code") or "").strip()
|
||
if not node_code:
|
||
return _agent_error("node_code 不能为空", "agent_node_code_required")
|
||
ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code)
|
||
if not ok:
|
||
return False, message, _auth
|
||
client_event_id = str(payload.get("client_event_id") or "").strip()[:128]
|
||
event_payload = dict(payload.get("payload") or {})
|
||
occurred_at = str(payload.get("occurred_at") or "").strip()
|
||
summary_text = str(payload.get("summary_text") or "").strip()
|
||
focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {}
|
||
if occurred_at and "occurred_at" not in event_payload:
|
||
event_payload["occurred_at"] = occurred_at
|
||
if summary_text and "summary_text" not in event_payload:
|
||
event_payload["summary_text"] = summary_text
|
||
if focus_ref and "focus_ref" not in event_payload:
|
||
event_payload["focus_ref"] = focus_ref
|
||
event_id = append_ops_job_event(
|
||
job_id=int(job_id),
|
||
step_id=int(payload.get("step_id") or 0) or None,
|
||
node_code=node_code,
|
||
client_event_id=client_event_id,
|
||
event_type=str(payload.get("event_type") or "agent_event").strip() or "agent_event",
|
||
level=str(payload.get("level") or "info").strip() or "info",
|
||
message=str(payload.get("message") or "").strip()[:2000] or "agent event",
|
||
payload=event_payload,
|
||
)
|
||
return True, "事件已写入", {
|
||
"event_id": event_id,
|
||
"client_event_id": client_event_id,
|
||
"event": get_ops_job_event(event_id),
|
||
}
|