from __future__ import annotations import hashlib import json import secrets import shlex import threading from datetime import datetime, timedelta from app.core.config import settings from app.core.db import get_db from app.services.ops_command_service import build_bash_command from app.services.ops_job_service import create_ops_job, ensure_ops_schema, get_ops_job from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template from app.services.runtime_settings_service import get_runtime_settings from app.services.sensitive_words_service import get_sensitive_words_payload from app.services.settings_service import get_settings_payload _AGENT_SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS ops_node_tokens ( id BIGSERIAL PRIMARY KEY, token_hash VARCHAR(128) NOT NULL UNIQUE, node_code VARCHAR(64) NOT NULL DEFAULT '', purpose VARCHAR(32) NOT NULL DEFAULT 'agent', issued_by VARCHAR(64) NOT NULL DEFAULT 'system', is_enabled BOOLEAN NOT NULL DEFAULT TRUE, expires_at TIMESTAMP, last_used_at TIMESTAMP, metadata_json JSONB, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_ops_node_tokens_node_code ON ops_node_tokens(node_code, is_enabled, purpose); ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT ''; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) NOT NULL DEFAULT 'agent'; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS issued_by VARCHAR(64) NOT NULL DEFAULT 'system'; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN NOT NULL DEFAULT TRUE; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS last_used_at TIMESTAMP; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS metadata_json JSONB; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; CREATE TABLE IF NOT EXISTS ops_job_events ( id BIGSERIAL PRIMARY KEY, job_id BIGINT REFERENCES ops_jobs(id) ON DELETE CASCADE, step_id BIGINT REFERENCES ops_job_steps(id) ON DELETE SET NULL, node_code VARCHAR(64) NOT NULL DEFAULT '', client_event_id VARCHAR(128) NOT NULL DEFAULT '', event_type VARCHAR(64) NOT NULL DEFAULT '', level VARCHAR(16) NOT NULL DEFAULT 'info', message TEXT NOT NULL DEFAULT '', payload_json JSONB, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS job_id BIGINT; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS step_id BIGINT; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT ''; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS client_event_id VARCHAR(128) NOT NULL DEFAULT ''; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS event_type VARCHAR(64) NOT NULL DEFAULT ''; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS level VARCHAR(16) NOT NULL DEFAULT 'info'; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS message TEXT NOT NULL DEFAULT ''; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS payload_json JSONB; ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; CREATE INDEX IF NOT EXISTS idx_ops_job_events_job_created ON ops_job_events(job_id, created_at DESC); CREATE UNIQUE INDEX IF NOT EXISTS uq_ops_job_events_job_client_event ON ops_job_events(job_id, client_event_id) WHERE client_event_id <> ''; ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS last_agent_complete_request_id VARCHAR(128) NOT NULL DEFAULT ''; """ _OPS_AGENT_SCHEMA_LOCK = threading.Lock() _OPS_AGENT_SCHEMA_READY = False _OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY = 90421802 def ensure_ops_agent_schema() -> None: global _OPS_AGENT_SCHEMA_READY ensure_ops_schema() if _OPS_AGENT_SCHEMA_READY: return with _OPS_AGENT_SCHEMA_LOCK: if _OPS_AGENT_SCHEMA_READY: return with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,)) cur.execute(_AGENT_SCHEMA_SQL) conn.commit() _OPS_AGENT_SCHEMA_READY = True def _hash_token(token: str) -> str: return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest() def _decode_json(value: object) -> dict: if isinstance(value, dict): return value if value in (None, ""): return {} try: return json.loads(value) except Exception: return {} def _format_time(value: object) -> str: if isinstance(value, datetime): return value.isoformat(sep=" ", timespec="seconds") return "" def _prefer_non_loopback_identity(primary: object, fallback: object) -> str: primary_value = str(primary or "").strip() fallback_value = str(fallback or "").strip() invalid_values = {"", "localhost", "localhost.localdomain", "127.0.0.1", "::1"} if primary_value and primary_value.lower() not in invalid_values: return primary_value return fallback_value def _merge_detect_runtime_snapshot(cluster_metadata: dict, metadata: dict, current_load: int) -> dict: active_threads = int(cluster_metadata.get("active_threads", metadata.get("active_threads", 0)) or 0) max_threads = int(cluster_metadata.get("max_threads", metadata.get("max_threads", 0)) or 0) inferred_worker_online = bool( cluster_metadata.get("worker_online", metadata.get("worker_online", False)) or cluster_metadata.get("service_running", metadata.get("service_running", False)) or current_load > 0 or active_threads > 0 ) inferred_detect_participating = bool( cluster_metadata.get("detect_participating", metadata.get("detect_participating", False)) or current_load > 0 or active_threads > 0 ) return { "worker_online": inferred_worker_online, "detect_participating": inferred_detect_participating, "active_threads": active_threads, "max_threads": max_threads, "current_load": current_load, "phase_label": str(cluster_metadata.get("phase_label", metadata.get("phase_label", "")) or "").strip(), "phase_detail": str(cluster_metadata.get("phase_detail", metadata.get("phase_detail", "")) or "").strip(), "recent_warning": str(cluster_metadata.get("recent_warning", metadata.get("recent_warning", "")) or "").strip(), "proxy_runtime_label": str( cluster_metadata.get("proxy_runtime_label", metadata.get("proxy_runtime_label", "")) or "" ).strip(), "proxy_runtime_reason": str( cluster_metadata.get("proxy_runtime_reason", metadata.get("proxy_runtime_reason", "")) or "" ).strip(), "updated_at": str(cluster_metadata.get("updated_at", metadata.get("updated_at", "")) or "").strip(), } def _ops_job_event_level_label(level: object) -> str: normalized_level = str(level or "").strip().lower() mapping = { "debug": "调试", "info": "信息", "success": "成功", "warning": "警告", "error": "错误", "critical": "严重", } return mapping.get(normalized_level, normalized_level or "信息") def _build_ops_job_event_row(row: tuple) -> dict: created_at = _format_time(row[9]) payload = _decode_json(row[8]) job_id = int(row[1]) if row[1] else 0 event_id = int(row[0]) event_type = str(row[5] or "") level = str(row[6] or "") message = str(row[7] or "") occurred_at = str(payload.get("occurred_at") or "").strip() or created_at summary_text = str(payload.get("summary_text") or "").strip() or message focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} if not focus_ref: focus_ref = { "kind": "ops_job_event", "job_id": job_id, "focus_event_id": event_id, "event_type": event_type, } return { "id": event_id, "event_key": f"job-event:{event_id}", "job_id": job_id, "step_id": int(row[2]) if row[2] else 0, "node_code": str(row[3] or ""), "client_event_id": str(row[4] or ""), "event_type": event_type, "level": level, "level_label": _ops_job_event_level_label(level), "message": message, "summary": message, "summary_text": summary_text, "payload": payload, "has_payload": bool(payload), "created_at": created_at, "occurred_at": occurred_at, "focus_ref": focus_ref, "ui_intent": { "kind": "job_detail", "job_id": job_id, "focus_event_id": event_id, "event_type": event_type, }, } def summarize_ops_job_events(events: list[dict], *, job_id: int = 0, limit: int = 50) -> dict: level_counts: dict[str, int] = {} event_type_counts: dict[str, int] = {} node_counts: dict[str, int] = {} latest_at = "" for event in list(events or []): event_level = str(event.get("level") or "").strip() or "info" event_type = str(event.get("event_type") or "").strip() or "event" node_code = str(event.get("node_code") or "").strip() occurred_at = str(event.get("occurred_at") or event.get("created_at") or "").strip() level_counts[event_level] = int(level_counts.get(event_level, 0) or 0) + 1 event_type_counts[event_type] = int(event_type_counts.get(event_type, 0) or 0) + 1 if node_code: node_counts[node_code] = int(node_counts.get(node_code, 0) or 0) + 1 if occurred_at and occurred_at > latest_at: latest_at = occurred_at return { "job_id": int(job_id or 0), "returned_total": len(events or []), "level_counts": level_counts, "event_type_counts": event_type_counts, "node_counts": node_counts, "latest_at": latest_at, "filters": { "limit": int(limit or 50), }, } def _parse_time(value: object) -> datetime | None: if isinstance(value, datetime): return value raw_value = str(value or "").strip() if not raw_value: return None normalized = raw_value.replace("Z", "+00:00") try: return datetime.fromisoformat(normalized) except Exception: pass for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): try: return datetime.strptime(raw_value, fmt) except ValueError: continue return None def _agent_error(message: str, detail_code: str, data: dict | None = None) -> tuple[bool, str, dict]: payload = dict(data or {}) payload["detail_code"] = str(detail_code or "").strip() return False, message, payload def _agent_state_from_context( *, last_seen_at: str, latest_token: dict, cluster_status: str, current_load: int, ) -> tuple[str, str, str]: now = datetime.now() last_seen_dt = _parse_time(last_seen_at) token_status = str(latest_token.get("token_status") or "absent").strip() or "absent" if last_seen_dt: current_time = datetime.now(last_seen_dt.tzinfo) if last_seen_dt.tzinfo else now age_seconds = max(0, int((current_time - last_seen_dt).total_seconds())) if age_seconds <= 180: if cluster_status == "busy" or int(current_load or 0) > 0: return "online_busy", "已接管", f"Agent 最近心跳 {age_seconds} 秒前,节点当前负载 {int(current_load or 0)}。" return "online", "已接管", f"Agent 最近心跳 {age_seconds} 秒前。" return "stale", "心跳过期", f"Agent 最近心跳 {age_seconds} 秒前,已超过在线阈值。" if token_status == "active": return "pending_bootstrap", "待接入", "已签发有效 Agent Token,但节点尚未完成 register/heartbeat。" if token_status == "expired": return "token_expired", "Token 过期", "已签发 Agent Token,但该 Token 已过期。" if token_status == "disabled": return "token_disabled", "Token 已禁用", "最近一枚 Agent Token 已被禁用,需重新签发。" if cluster_status in {"online", "busy"}: return "runtime_only", "仅运行态在线", "控制面能看到 runtime 心跳,但 Node Agent 尚未接管。" return "unmanaged", "未接管", "当前节点尚未生成有效 Agent 接入状态。" def _ssh_access_context(node: dict) -> tuple[bool, str, str]: ssh_host = str(node.get("ssh_host") or "").strip() ssh_user = str(node.get("ssh_user") or "").strip() if ssh_host and ssh_user: return True, "SSH 已备好", f"已保存 SSH 入口 {ssh_user}@{ssh_host},后续可由海外控制面 SSH 执行器接管。" if ssh_host and not ssh_user: return False, "SSH 待完善", "已填写 SSH 主机,但还缺少 ssh_user。" if ssh_user and not ssh_host: return False, "SSH 待完善", "已填写 ssh_user,但还缺少 ssh_host。" return False, "SSH 未配置", "当前还没有保存 SSH 入口。" def _remote_access_context(node: dict) -> tuple[str, str, str, bool]: is_managed = bool(node.get("is_managed", False)) is_enabled = bool(node.get("is_enabled", False)) is_agent_online = bool(node.get("is_agent_online", False)) ssh_ready, ssh_label, ssh_reason = _ssh_access_context(node) if not is_managed: return "unmanaged", "未纳管", "当前节点还没有进入海外控制面的托管清单。", False if not is_enabled: return "disabled", "已停用", "当前节点已纳管,但被标记为停用。", False if is_agent_online and ssh_ready: return "hybrid_ready", "Agent + SSH", "Node Agent 已在线,SSH 入口也已备好。", True if is_agent_online: return "agent_ready", "Agent 在线", "当前节点已经进入标准自动执行器就绪状态。", True if ssh_ready: return ( "ssh_ready", "SSH 可执行", f"{ssh_reason} 当前已可执行日志、诊断和部分服务控制;正式发布与标准 Rollout 仍优先依赖 Node Agent。", True, ) return "agent_pending", "待接入", "当前还没有可用 Agent,也没有保存 SSH 入口。", False def _delivery_queue_context(metadata: dict) -> tuple[str, str, str, dict]: snapshot = dict((metadata or {}).get("delivery_queue") or {}) pending_count = int(snapshot.get("pending_count", 0) or 0) dead_letter_count = int(snapshot.get("dead_letter_count", 0) or 0) last_flush_at = str(snapshot.get("last_flush_at") or "").strip() oldest_pending_at = str(snapshot.get("oldest_pending_at") or "").strip() oldest_dead_letter_at = str(snapshot.get("oldest_dead_letter_at") or "").strip() if not snapshot: return "unknown", "未上报", "Node Agent 尚未上报回执队列状态。", {} if dead_letter_count > 0: reason = f"当前存在 {dead_letter_count} 条死信记录,建议优先查看节点日志或诊断编排。" if oldest_dead_letter_at: reason += f" 最早死信时间:{oldest_dead_letter_at}。" return "dead_letter", f"死信 {dead_letter_count}", reason, snapshot if pending_count > 0: reason = f"当前存在 {pending_count} 条待重试回执,Node Agent 会继续自动回放。" if oldest_pending_at: reason += f" 最早积压时间:{oldest_pending_at}。" return "retrying", f"待重试 {pending_count}", reason, snapshot reason = "当前没有待重试回执,也没有死信记录。" if last_flush_at: reason += f" 最近一次队列冲刷:{last_flush_at}。" return "healthy", "正常", reason, snapshot def _summarize_managed_node_states(nodes: list[dict]) -> dict: status_counts: dict[str, int] = {} remote_access_counts: dict[str, int] = {} delivery_queue_state_counts: dict[str, int] = {} for item in nodes: state = str(item.get("agent_state") or "").strip() or "unknown" status_counts[state] = int(status_counts.get(state, 0) or 0) + 1 remote_state = str(item.get("remote_access_state") or "").strip() or "unknown" remote_access_counts[remote_state] = int(remote_access_counts.get(remote_state, 0) or 0) + 1 queue_state = str(item.get("delivery_queue_state") or "").strip() or "unknown" delivery_queue_state_counts[queue_state] = int(delivery_queue_state_counts.get(queue_state, 0) or 0) + 1 managed_nodes = [item for item in nodes if bool(item.get("is_managed", False))] participating_nodes = [item for item in nodes if bool(item.get("is_current_participant", False))] dispatch_active_nodes = [item for item in nodes if bool(item.get("is_dispatch_active", False))] recent_only_nodes = [ item for item in nodes if str(item.get("participation_state") or "").strip() == "recent_throughput" ] standby_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "standby"] load_syncing_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "load_syncing"] return { "status_counts": status_counts, "online": int(status_counts.get("online", 0) or 0) + int(status_counts.get("online_busy", 0) or 0), "stale": int(status_counts.get("stale", 0) or 0), "pending_bootstrap": int(status_counts.get("pending_bootstrap", 0) or 0), "runtime_only": int(status_counts.get("runtime_only", 0) or 0), "unmanaged": int(status_counts.get("unmanaged", 0) or 0), "token_issue": int(status_counts.get("token_expired", 0) or 0) + int(status_counts.get("token_disabled", 0) or 0), "ssh_ready": int(remote_access_counts.get("ssh_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0), "agent_ready": int(remote_access_counts.get("agent_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0), "remote_access_ready": sum(1 for item in nodes if bool(item.get("is_remote_access_ready", False))), "managed_total": len(managed_nodes), "managed_enabled": sum(1 for item in managed_nodes if bool(item.get("is_enabled", False))), "queue_retrying_nodes": sum( 1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "retrying" ), "queue_dead_letter_nodes": sum( 1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "dead_letter" ), "queue_pending_records": sum(int(item.get("delivery_queue_pending_count", 0) or 0) for item in managed_nodes), "queue_dead_letter_records": sum( int(item.get("delivery_queue_dead_letter_count", 0) or 0) for item in managed_nodes ), "participating": len(participating_nodes), "dispatch_active": len(dispatch_active_nodes), "recent_only": len(recent_only_nodes), "standby": len(standby_nodes), "load_syncing": len(load_syncing_nodes), "non_participating": len(standby_nodes) + len(load_syncing_nodes), "total": len(nodes), "remote_access_state_counts": remote_access_counts, "delivery_queue_state_counts": delivery_queue_state_counts, } def _normalize_detect_participation_rows(payload: dict | None = None) -> dict[str, dict]: normalized_payload = dict(payload or {}) if "detect" in normalized_payload and isinstance(normalized_payload.get("detect"), dict): normalized_payload = dict(normalized_payload.get("detect") or {}) rows: dict[str, dict] = {} for collection_key in ("participating_nodes", "non_participating_nodes", "standby_nodes"): for item in list(normalized_payload.get(collection_key) or []): node_code = str((item or {}).get("node_code") or "").strip() if not node_code: continue rows[node_code] = dict(item or {}) return rows def append_ops_job_event( *, job_id: int | None, step_id: int | None = None, node_code: str = "", client_event_id: str = "", event_type: str, message: str, level: str = "info", payload: dict | None = None, ) -> int: ensure_ops_agent_schema() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO ops_job_events ( job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) ON CONFLICT (job_id, client_event_id) WHERE client_event_id <> '' DO UPDATE SET step_id = COALESCE(EXCLUDED.step_id, ops_job_events.step_id), node_code = CASE WHEN EXCLUDED.node_code <> '' THEN EXCLUDED.node_code ELSE ops_job_events.node_code END, event_type = EXCLUDED.event_type, level = EXCLUDED.level, message = EXCLUDED.message, payload_json = EXCLUDED.payload_json RETURNING id """, ( int(job_id) if job_id else None, int(step_id) if step_id else None, str(node_code or "").strip(), str(client_event_id or "").strip()[:128], str(event_type or "").strip() or "event", str(level or "info").strip() or "info", str(message or "").strip()[:2000], json.dumps(payload or {}, ensure_ascii=False), ), ) event_id = int(cur.fetchone()[0]) conn.commit() return event_id def list_ops_job_events(job_id: int, limit: int = 50) -> list[dict]: ensure_ops_agent_schema() safe_limit = min(max(int(limit or 50), 1), 200) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at FROM ops_job_events WHERE job_id = %s ORDER BY id DESC LIMIT %s """, (int(job_id), safe_limit), ) rows = cur.fetchall() return [_build_ops_job_event_row(row) for row in rows] def get_ops_job_event(event_id: int) -> dict: ensure_ops_agent_schema() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at FROM ops_job_events WHERE id = %s LIMIT 1 """, (int(event_id),), ) row = cur.fetchone() return _build_ops_job_event_row(row) if row else {} def list_ops_job_events_for_jobs(job_ids: list[int], limit: int = 120) -> list[dict]: ensure_ops_agent_schema() normalized_job_ids = sorted({int(item) for item in list(job_ids or []) if int(item or 0) > 0}) if not normalized_job_ids: return [] safe_limit = min(max(int(limit or 120), 1), 800) placeholders = ", ".join(["%s"] * len(normalized_job_ids)) with get_db() as conn: with conn.cursor() as cur: cur.execute( f""" SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at FROM ops_job_events WHERE job_id IN ({placeholders}) ORDER BY id DESC LIMIT %s """, (*normalized_job_ids, safe_limit), ) rows = cur.fetchall() return [_build_ops_job_event_row(row) for row in rows] def list_managed_nodes_with_agent_state(*, participation_payload: dict | None = None) -> dict: from app.services.cluster_runtime_service import get_cluster_snapshot from app.services.ops_job_service import list_managed_nodes ensure_ops_agent_schema() managed_nodes = list_managed_nodes() cluster_nodes = list((get_cluster_snapshot().get("nodes") or [])) cluster_map = { str(item.get("node_code") or "").strip(): item for item in cluster_nodes if str(item.get("node_code") or "").strip() } detect_participation_map = _normalize_detect_participation_rows(participation_payload) if not detect_participation_map: try: from app.services.runtime_status_service import get_runtime_status runtime_payload = get_runtime_status() detect_participation_map = _normalize_detect_participation_rows(runtime_payload.get("detect") or {}) except Exception: detect_participation_map = {} latest_tokens: dict[str, dict] = {} latest_jobs: dict[str, dict] = {} with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT DISTINCT ON (node_code) node_code, id, issued_by, is_enabled, expires_at, last_used_at, metadata_json, created_at, updated_at FROM ops_node_tokens WHERE purpose = 'agent' ORDER BY node_code ASC, created_at DESC, id DESC """ ) for row in cur.fetchall(): node_code = str(row[0] or "").strip() if not node_code: continue expires_at = _format_time(row[4]) token_status = "active" expires_dt = _parse_time(row[4]) if not bool(row[3]): token_status = "disabled" elif expires_dt: current_time = datetime.now(expires_dt.tzinfo) if expires_dt.tzinfo else datetime.now() if expires_dt < current_time: token_status = "expired" latest_tokens[node_code] = { "record_id": int(row[1]), "issued_by": str(row[2] or ""), "is_enabled": bool(row[3]), "expires_at": expires_at, "last_used_at": _format_time(row[5]), "metadata": _decode_json(row[6]), "created_at": _format_time(row[7]), "updated_at": _format_time(row[8]), "token_status": token_status, } cur.execute( """ SELECT DISTINCT ON (target_node_code) target_node_code, id, job_code, action, status, execution_mode, created_at, updated_at FROM ops_jobs WHERE target_node_code <> '' ORDER BY target_node_code ASC, created_at DESC, id DESC """ ) for row in cur.fetchall(): node_code = str(row[0] or "").strip() if not node_code: continue latest_jobs[node_code] = { "id": int(row[1]), "job_code": str(row[2] or ""), "action": str(row[3] or ""), "status": str(row[4] or ""), "execution_mode": str(row[5] or ""), "created_at": _format_time(row[6]), "updated_at": _format_time(row[7]), } enriched_nodes: list[dict] = [] seen_node_codes: set[str] = set() for node in managed_nodes: node_code = str(node.get("node_code") or "").strip() if not node_code: continue seen_node_codes.add(node_code) metadata = dict(node.get("metadata") or {}) cluster_node = cluster_map.get(node_code, {}) cluster_metadata = dict(cluster_node.get("metadata") or {}) latest_token = latest_tokens.get(node_code, {}) latest_job = latest_jobs.get(node_code, {}) participation_row = dict(detect_participation_map.get(node_code) or {}) last_seen_at = str(node.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip() cluster_status = str(cluster_node.get("status") or metadata.get("cluster_status") or "").strip() current_load = int(cluster_node.get("current_load", 0) or 0) ( delivery_queue_state, delivery_queue_label, delivery_queue_reason, delivery_queue_snapshot, ) = _delivery_queue_context(metadata) agent_state, agent_state_label, agent_state_reason = _agent_state_from_context( last_seen_at=last_seen_at, latest_token=latest_token, cluster_status=cluster_status, current_load=current_load, ) capabilities = list(metadata.get("capabilities") or []) detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load) active_job_snapshot = { "job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(), "status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(), "items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0), "items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0), "items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0), "items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0), } merged_node = { **node, "is_managed": True, "last_seen_at": last_seen_at, "agent_state": agent_state, "agent_state_label": agent_state_label, "agent_state_reason": agent_state_reason, "is_agent_online": agent_state in {"online", "online_busy"}, "capabilities": capabilities, "capabilities_count": len(capabilities), "agent_version": str(metadata.get("agent_version") or "").strip(), "agent_hostname": str(metadata.get("hostname") or "").strip(), "agent_ip": str(metadata.get("ip") or "").strip(), "cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")), "cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")), "cluster_status": cluster_status, "cluster_current_load": current_load, "current_load": current_load, "detect_runtime": detect_runtime, "runtime_state": detect_runtime, "active_job": active_job_snapshot, "cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or metadata.get("last_heartbeat_at") or "").strip(), "cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", metadata.get("is_effective_worker", False))), "cluster_detect_participating": bool( participation_row.get("is_current_participant", False) or cluster_node.get("detect_participating", metadata.get("detect_participating", False)) ), "participation_state": str(participation_row.get("participation_state") or "").strip(), "participation_label": str(participation_row.get("participation_label") or "").strip(), "participation_reason": str(participation_row.get("participation_reason") or "").strip(), "is_current_participant": bool(participation_row.get("is_current_participant", False)), "is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)), "items_total": int(participation_row.get("items_total", 0) or 0), "items_pending": int(participation_row.get("items_pending", 0) or 0), "items_claimed": int(participation_row.get("items_claimed", 0) or 0), "items_running": int(participation_row.get("items_running", 0) or 0), "items_completed": int(participation_row.get("items_completed", 0) or 0), "items_failed": int(participation_row.get("items_failed", 0) or 0), "processed_recent": int(participation_row.get("processed_recent", 0) or 0), "processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0), "delivery_queue_state": delivery_queue_state, "delivery_queue_label": delivery_queue_label, "delivery_queue_reason": delivery_queue_reason, "delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0), "delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0), "delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(), "delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(), "delivery_queue_oldest_pending_request_id": str( delivery_queue_snapshot.get("oldest_pending_request_id") or "" ).strip(), "delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(), "delivery_queue_oldest_dead_letter_at": str( delivery_queue_snapshot.get("oldest_dead_letter_at") or "" ).strip(), "delivery_queue_oldest_dead_letter_request_id": str( delivery_queue_snapshot.get("oldest_dead_letter_request_id") or "" ).strip(), "delivery_queue_oldest_dead_letter_kind": str( delivery_queue_snapshot.get("oldest_dead_letter_kind") or "" ).strip(), "latest_token": latest_token, "latest_job": latest_job, } ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(merged_node) remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context( merged_node ) enriched_nodes.append( { **merged_node, "has_ssh_access": ssh_ready, "ssh_access_label": ssh_access_label, "ssh_access_reason": ssh_access_reason, "remote_access_state": remote_access_state, "remote_access_label": remote_access_label, "remote_access_reason": remote_access_reason, "is_remote_access_ready": is_remote_access_ready, } ) for node_code, cluster_node in cluster_map.items(): if not node_code or node_code in seen_node_codes: continue metadata = dict(cluster_node.get("metadata") or {}) cluster_metadata = dict(cluster_node.get("metadata") or {}) latest_token = latest_tokens.get(node_code, {}) latest_job = latest_jobs.get(node_code, {}) participation_row = dict(detect_participation_map.get(node_code) or {}) cluster_status = str(cluster_node.get("status") or "").strip() current_load = int(cluster_node.get("current_load", 0) or 0) ( delivery_queue_state, delivery_queue_label, delivery_queue_reason, delivery_queue_snapshot, ) = _delivery_queue_context(metadata) agent_state, agent_state_label, agent_state_reason = _agent_state_from_context( last_seen_at="", latest_token=latest_token, cluster_status=cluster_status, current_load=current_load, ) capabilities = list(metadata.get("capabilities") or []) detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load) active_job_snapshot = { "job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(), "status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(), "items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0), "items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0), "items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0), "items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0), } fallback_node = { "node_code": node_code, "region": str(cluster_node.get("region") or "").strip(), "role": str(cluster_node.get("role") or "").strip(), "title": str(cluster_node.get("hostname") or node_code).strip(), "ssh_host": "", "ssh_port": 22, "ssh_user": "", "auth_mode": "key", "deploy_channel": "", "is_enabled": False, "is_managed": False, "metadata": { **metadata, "imported_from_cluster_fallback": True, }, "created_at": "", "last_seen_at": "", "updated_at": "", "agent_state": agent_state, "agent_state_label": agent_state_label, "agent_state_reason": agent_state_reason, "is_agent_online": agent_state in {"online", "online_busy"}, "capabilities": capabilities, "capabilities_count": len(capabilities), "agent_version": str(metadata.get("agent_version") or "").strip(), "agent_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")), "agent_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")), "cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")), "cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")), "cluster_status": cluster_status, "cluster_current_load": current_load, "current_load": current_load, "detect_runtime": detect_runtime, "runtime_state": detect_runtime, "active_job": active_job_snapshot, "cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or "").strip(), "cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", False)), "cluster_detect_participating": bool( participation_row.get("is_current_participant", False) or cluster_node.get("detect_participating", False) ), "participation_state": str(participation_row.get("participation_state") or "").strip(), "participation_label": str(participation_row.get("participation_label") or "").strip(), "participation_reason": str(participation_row.get("participation_reason") or "").strip(), "is_current_participant": bool(participation_row.get("is_current_participant", False)), "is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)), "items_total": int(participation_row.get("items_total", 0) or 0), "items_pending": int(participation_row.get("items_pending", 0) or 0), "items_claimed": int(participation_row.get("items_claimed", 0) or 0), "items_running": int(participation_row.get("items_running", 0) or 0), "items_completed": int(participation_row.get("items_completed", 0) or 0), "items_failed": int(participation_row.get("items_failed", 0) or 0), "processed_recent": int(participation_row.get("processed_recent", 0) or 0), "processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0), "delivery_queue_state": delivery_queue_state, "delivery_queue_label": delivery_queue_label, "delivery_queue_reason": delivery_queue_reason, "delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0), "delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0), "delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(), "delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(), "delivery_queue_oldest_pending_request_id": str( delivery_queue_snapshot.get("oldest_pending_request_id") or "" ).strip(), "delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(), "delivery_queue_oldest_dead_letter_at": str( delivery_queue_snapshot.get("oldest_dead_letter_at") or "" ).strip(), "delivery_queue_oldest_dead_letter_request_id": str( delivery_queue_snapshot.get("oldest_dead_letter_request_id") or "" ).strip(), "delivery_queue_oldest_dead_letter_kind": str( delivery_queue_snapshot.get("oldest_dead_letter_kind") or "" ).strip(), "latest_token": latest_token, "latest_job": latest_job, } ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(fallback_node) remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context( fallback_node ) enriched_nodes.append( { **fallback_node, "has_ssh_access": ssh_ready, "ssh_access_label": ssh_access_label, "ssh_access_reason": ssh_access_reason, "remote_access_state": remote_access_state, "remote_access_label": remote_access_label, "remote_access_reason": remote_access_reason, "is_remote_access_ready": is_remote_access_ready, } ) enriched_nodes.sort( key=lambda item: ( 0 if bool(item.get("is_managed", False)) else 1, str(item.get("region") or ""), str(item.get("role") or ""), str(item.get("node_code") or ""), ) ) return { "nodes": enriched_nodes, "summary": _summarize_managed_node_states(enriched_nodes), } def _build_managed_node_handover_stage(node: dict) -> dict: normalized_node = dict(node or {}) node_code = str(normalized_node.get("node_code") or "").strip() agent_state = str(normalized_node.get("agent_state") or "").strip() agent_state_label = str(normalized_node.get("agent_state_label") or "").strip() or "未知" agent_state_reason = str(normalized_node.get("agent_state_reason") or "").strip() remote_access_state = str(normalized_node.get("remote_access_state") or "").strip() remote_access_label = str(normalized_node.get("remote_access_label") or "").strip() or "未就绪" remote_access_reason = str(normalized_node.get("remote_access_reason") or "").strip() token_status = str(((normalized_node.get("latest_token") or {}).get("token_status") or "")).strip() is_managed = bool(normalized_node.get("is_managed", False)) is_enabled = bool(normalized_node.get("is_enabled", False)) has_ssh_access = bool(normalized_node.get("has_ssh_access", False)) if not is_managed: return { "code": "unmanaged", "label": "未纳管", "summary": "节点已出现在集群视图,但还没保存到海外控制面的托管清单。", "next_step": { "code": "save_managed_node", "label": "先保存节点资料", "summary": "先补齐 SSH 入口、身份和部署通道,再生成 Node Agent 接入方案。", "ui_intent": { "kind": "managed_node_edit", "node_code": node_code, }, }, } if not is_enabled: return { "code": "disabled", "label": "已停用", "summary": "节点虽然已经纳管,但当前被标记为停用,标准运维动作不会继续下发。", "next_step": { "code": "enable_managed_node", "label": "检查并重新启用", "summary": "确认该节点仍应参与运维后,先在托管节点面板重新启用。", "ui_intent": { "kind": "managed_node_edit", "node_code": node_code, }, }, } if agent_state in {"online", "online_busy"}: return { "code": "ready", "label": "接管完成", "summary": "Node Agent 已经在线,节点已经进入标准自动执行链路。", "next_step": { "code": "run_acceptance", "label": "执行接管验收", "summary": "建议立刻跑一轮 onboarding.acceptance 或标准巡检,确认日志、健康快照和发布链路都打通。", "ui_intent": { "kind": "open_playbook_dialog", "playbook_key": "onboarding.acceptance", "target_node_codes": [node_code] if node_code else [], "execution_mode": "remote-agent", "auto_approve": True, }, }, } if agent_state == "stale": return { "code": "stale", "label": "心跳过期", "summary": "节点曾经接入过,但 Node Agent 心跳已经过期,当前不能视为稳定可控。", "next_step": { "code": "recover_agent", "label": "优先恢复 Agent", "summary": "先检查 domaincheck-node-agent 服务、网络连通性和控制面地址,再决定是否重新签发接入方案。", "ui_intent": { "kind": "managed_node_handover", "node_code": node_code, }, }, } if token_status in {"expired", "disabled"} or agent_state in {"token_expired", "token_disabled"}: return { "code": "token_issue", "label": "Token 失效", "summary": "最近一枚 Node Agent Token 已不可继续使用,需要重新生成接入方案。", "next_step": { "code": "reissue_bootstrap_plan", "label": "重新签发接入方案", "summary": "重新生成 bootstrap env 和一键落地片段,再到目标节点执行。", "ui_intent": { "kind": "managed_node_handover", "node_code": node_code, }, }, } if agent_state == "pending_bootstrap": return { "code": "pending_bootstrap", "label": "待执行接入", "summary": "控制面已经签发了有效 Token,当前卡在目标节点尚未执行 bootstrap 片段。", "next_step": { "code": "execute_bootstrap_plan", "label": "去目标机执行接入片段", "summary": "把 env、脚本或一键落地片段拷到目标机执行,然后观察 register / heartbeat 是否建立。", "ui_intent": { "kind": "managed_node_handover", "node_code": node_code, }, }, } if remote_access_state == "ssh_ready" or has_ssh_access: return { "code": "ssh_ready", "label": remote_access_label or "SSH 已备好", "summary": "SSH 入口已经可用,可以从海外控制面统一生成并下发 Node Agent 接入方案。", "next_step": { "code": "issue_bootstrap_plan", "label": "生成接入方案", "summary": "下一步就是签发 bootstrap env、脚本和一键落地命令,然后在目标节点启动 domaincheck-node-agent。", "ui_intent": { "kind": "managed_node_handover", "node_code": node_code, }, }, } if agent_state == "runtime_only": return { "code": "runtime_only", "label": agent_state_label or "仅运行态在线", "summary": "控制面能看到 runtime 心跳,但 Node Agent 还没进入标准接管链路。", "next_step": { "code": "complete_ssh_profile", "label": "补齐 SSH 入口并生成接入方案", "summary": "先把 SSH 主机和账号补齐,再从海外控制面统一签发 Node Agent 接入方案。", "ui_intent": { "kind": "managed_node_edit", "node_code": node_code, }, }, } return { "code": "profile_incomplete", "label": remote_access_label or agent_state_label or "待完善", "summary": remote_access_reason or agent_state_reason or "当前接管资料还不完整,尚不能进入标准自动执行链路。", "next_step": { "code": "complete_ssh_profile", "label": "补齐节点接入资料", "summary": "至少补齐 SSH 入口,再生成标准 Node Agent 接入方案。", "ui_intent": { "kind": "managed_node_edit", "node_code": node_code, }, }, } def _build_managed_node_handover_payload( *, node: dict, control_plane_base_url: str = "", root_dir: str = "", ) -> dict: normalized_node = dict(node or {}) normalized_node_code = str(normalized_node.get("node_code") or "").strip() if not normalized_node_code: return {} node = normalized_node metadata = dict(node.get("metadata") or {}) latest_token = dict(node.get("latest_token") or {}) latest_job = dict(node.get("latest_job") or {}) control_plane_value = _normalize_control_plane_base_url(control_plane_base_url) if not control_plane_value: control_plane_value = ( _normalize_control_plane_base_url(str(metadata.get("ops_control_plane_base_url") or "")) or f"http://127.0.0.1:{settings.api_port}" ) resolved_root_dir = ( str(root_dir or metadata.get("root_dir") or metadata.get("deploy_root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck" ) stage = _build_managed_node_handover_stage(node) blocking_items: list[str] = [] attention_items: list[str] = [] if not bool(node.get("is_managed", False)): blocking_items.append("当前节点还未纳入海外控制面的托管清单。") if bool(node.get("is_managed", False)) and not bool(node.get("is_enabled", False)): blocking_items.append("当前节点已纳管,但被标记为停用。") if not bool(node.get("has_ssh_access", False)): attention_items.append("当前还没有完整 SSH 入口,无法从海外控制面直接接管该节点。") agent_state = str(node.get("agent_state") or "").strip() if agent_state == "pending_bootstrap": blocking_items.append("已签发有效 Token,但目标节点尚未执行 bootstrap 片段。") elif agent_state == "stale": blocking_items.append("Node Agent 心跳已过期,需先恢复服务或重新接入。") elif agent_state in {"token_expired", "token_disabled"}: blocking_items.append("最近一枚 Node Agent Token 已失效,需要重新签发。") if str(node.get("delivery_queue_state") or "").strip() == "dead_letter": attention_items.append( f"当前节点存在 {int(node.get('delivery_queue_dead_letter_count', 0) or 0)} 条死信回执,建议先查看 delivery queue。" ) elif str(node.get("delivery_queue_state") or "").strip() == "retrying": attention_items.append( f"当前节点还有 {int(node.get('delivery_queue_pending_count', 0) or 0)} 条待重试回执,Node Agent 会继续自动回放。" ) return { "node_code": normalized_node_code, "summary": str(stage.get("summary") or "").strip(), "stage": stage, "node": node, "control_plane_base_url": control_plane_value, "root_dir": resolved_root_dir, "readiness": { "is_managed": bool(node.get("is_managed", False)), "is_enabled": bool(node.get("is_enabled", False)), "has_ssh_access": bool(node.get("has_ssh_access", False)), "is_agent_online": bool(node.get("is_agent_online", False)), "is_remote_access_ready": bool(node.get("is_remote_access_ready", False)), }, "ssh_profile": { "host": str(node.get("ssh_host") or "").strip(), "port": int(node.get("ssh_port", 22) or 22), "user": str(node.get("ssh_user") or "").strip(), "auth_mode": str(node.get("auth_mode") or "").strip(), "label": str(node.get("ssh_access_label") or "").strip(), "reason": str(node.get("ssh_access_reason") or "").strip(), "ssh_command_hint": ( f"ssh -p {int(node.get('ssh_port', 22) or 22)} " f"{str(node.get('ssh_user') or '').strip()}@{str(node.get('ssh_host') or '').strip()}" if str(node.get("ssh_host") or "").strip() and str(node.get("ssh_user") or "").strip() else "" ), }, "cluster": { "status": str(node.get("cluster_status") or "").strip(), "current_load": int(node.get("cluster_current_load", 0) or 0), "last_heartbeat_at": str(node.get("cluster_last_heartbeat_at") or "").strip(), "hostname": str(node.get("cluster_hostname") or "").strip(), "ip": str(node.get("cluster_ip") or "").strip(), "is_effective_worker": bool(node.get("cluster_is_effective_worker", False)), }, "participation": { "state": str(node.get("participation_state") or "").strip(), "label": str(node.get("participation_label") or "").strip(), "reason": str(node.get("participation_reason") or "").strip(), "is_current_participant": bool(node.get("is_current_participant", False)), "is_dispatch_active": bool(node.get("is_dispatch_active", False)), "items_claimed": int(node.get("items_claimed", 0) or 0), "items_running": int(node.get("items_running", 0) or 0), "items_completed": int(node.get("items_completed", 0) or 0), "processed_recent": int(node.get("processed_recent", 0) or 0), "processed_per_minute": float(node.get("processed_per_minute", 0) or 0), }, "agent_token": latest_token, "latest_job": latest_job, "delivery_queue": { "state": str(node.get("delivery_queue_state") or "").strip(), "label": str(node.get("delivery_queue_label") or "").strip(), "reason": str(node.get("delivery_queue_reason") or "").strip(), "pending_count": int(node.get("delivery_queue_pending_count", 0) or 0), "dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0), "last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(), }, "blocking_items": blocking_items, "attention_items": attention_items, "bootstrap_defaults": { "node_code": normalized_node_code, "node_region": str(node.get("region") or "mainland").strip() or "mainland", "node_role": str(node.get("role") or "worker").strip() or "worker", "control_plane_base_url": control_plane_value, "root_dir": resolved_root_dir, "expires_in_hours": 72, "issued_by": "api", "metadata": { "issued_from": "ops-managed-node-handover", "node_region": str(node.get("region") or "mainland").strip() or "mainland", "node_role": str(node.get("role") or "worker").strip() or "worker", "remote_access_state": str(node.get("remote_access_state") or "").strip(), "agent_state": agent_state, }, }, "endpoints": { "detail": f"/api/v1/ops/nodes/{normalized_node_code}/handover", "bootstrap_plan": f"/api/v1/ops/nodes/{normalized_node_code}/handover/bootstrap-plan", }, } def get_managed_node_handover( node_code: str, *, control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> dict: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return {} resolved_nodes_payload = dict(nodes_payload or {}) if not resolved_nodes_payload: resolved_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=participation_payload) nodes = list(resolved_nodes_payload.get("nodes") or []) node = next( (dict(item or {}) for item in nodes if str(item.get("node_code") or "").strip() == normalized_node_code), {}, ) if not node: return {} return _build_managed_node_handover_payload( node=node, control_plane_base_url=control_plane_base_url, root_dir=root_dir, ) def get_managed_node_onboarding( node_code: str, *, control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> dict: handover = get_managed_node_handover( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not handover: return {} node = dict(handover.get("node") or {}) stage = dict(handover.get("stage") or {}) readiness = dict(handover.get("readiness") or {}) normalized_node_code = str(handover.get("node_code") or node_code or "").strip() stage_code = str(stage.get("code") or "").strip() has_ssh_access = bool(readiness.get("has_ssh_access", False)) is_agent_online = bool(readiness.get("is_agent_online", False)) is_remote_access_ready = bool(readiness.get("is_remote_access_ready", False)) acceptance_execution_mode = "" acceptance_status_code = "blocked" acceptance_status_label = "暂不可验收" acceptance_summary = "当前还没进入标准接管就绪状态,暂不建议直接跑 onboarding.acceptance。" if is_agent_online: acceptance_execution_mode = "remote-agent" acceptance_status_code = "ready" acceptance_status_label = "可远端验收" acceptance_summary = "Node Agent 已在线,可以直接跑 onboarding.acceptance 做标准接管验收。" elif has_ssh_access: acceptance_execution_mode = "ssh" acceptance_status_code = "ssh_only" acceptance_status_label = "可走 SSH 验收" acceptance_summary = "当前 Agent 还没在线,但 SSH 已备好,可先走 SSH 模式做接管验收。" onboarding_stage_code = stage_code or "unknown" onboarding_stage_label = str(stage.get("label") or "").strip() or "待判断" onboarding_summary = str(handover.get("summary") or stage.get("summary") or "").strip() if acceptance_status_code in {"ready", "ssh_only"} and stage_code == "ready": onboarding_stage_code = "acceptance_ready" onboarding_stage_label = "待接管验收" onboarding_summary = acceptance_summary acceptance_payload = { "playbook_key": "onboarding.acceptance", "target_node_codes": [normalized_node_code] if normalized_node_code else [], "execution_mode": acceptance_execution_mode or "remote-agent", "auto_approve": True, } bootstrap_payload = { "node_code": normalized_node_code, "node_region": str(node.get("region") or "mainland").strip() or "mainland", "node_role": str(node.get("role") or "worker").strip() or "worker", "control_plane_base_url": str(handover.get("control_plane_base_url") or "").strip(), "root_dir": str(handover.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck", } next_actions = [] if stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale"}: next_actions.append( { "key": "bootstrap_plan", "label": "生成接入方案", "summary": "先生成并执行 Node Agent bootstrap 方案,再观察 register / heartbeat 是否建立。", "command_hint": ( build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", "http://127.0.0.1:8100", normalized_node_code) if normalized_node_code else "" ), } ) next_actions.append( { "key": "bootstrap_run", "label": "签发接入工单", "summary": "把本节点 bootstrap 收编进 ops playbook / ops job,形成标准可追踪的纳管回执。", "command_hint": ( build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code) if normalized_node_code else "" ), } ) if acceptance_status_code in {"ready", "ssh_only"}: next_actions.append( { "key": "run_acceptance", "label": "执行接管验收", "summary": acceptance_summary, "command_hint": ( "通过 driver / runbook 执行 onboarding.acceptance,确认 health / node-agent / worker 三段验收全部通过。" ), } ) recovery_action = "noop" recovery_label = "当前无需额外恢复动作" recovery_summary = onboarding_summary or "当前节点暂无需要执行的接管恢复动作。" recovery_command_hint = "" recovery_needs_confirmation = True recovery_window = "none" if onboarding_stage_code == "acceptance_ready" or is_agent_online: recovery_action = "run_acceptance" recovery_label = "执行接管验收" recovery_summary = acceptance_summary or "当前节点已经进入验收窗口,建议直接执行 onboarding.acceptance。" recovery_command_hint = ( build_bash_command("drive_ops_center.sh", "node-acceptance-run", "http://127.0.0.1:8100", normalized_node_code, "cli") if normalized_node_code else "" ) recovery_window = "acceptance" elif onboarding_stage_code in { "pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale", "agent_pending", "unknown", "", }: recovery_action = "bootstrap_run" recovery_label = "签发接入工单" recovery_summary = "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。" recovery_command_hint = ( build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code, "cli") if normalized_node_code else "" ) recovery_window = "bootstrap" return { "node_code": normalized_node_code, "summary": onboarding_summary, "onboarding_stage": { "code": onboarding_stage_code, "label": onboarding_stage_label, "summary": onboarding_summary, }, "handover": handover, "acceptance": { "status_code": acceptance_status_code, "status_label": acceptance_status_label, "summary": acceptance_summary, "execution_mode": acceptance_execution_mode, "is_agent_online": is_agent_online, "has_ssh_access": has_ssh_access, "is_remote_access_ready": is_remote_access_ready, "playbook": acceptance_payload, }, "recommended_actions": next_actions, "recovery_decision": { "action": recovery_action, "label": recovery_label, "summary": recovery_summary, "command_hint": recovery_command_hint, "needs_confirmation": recovery_needs_confirmation, "window": recovery_window, "stage_code": onboarding_stage_code, "acceptance_status_code": acceptance_status_code, }, "bootstrap_plan_request": bootstrap_payload, } def preview_managed_node_onboarding_bootstrap( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {}) normalized_requested_by = str(requested_by or "api").strip() or "api" from app.services.ops_playbook_service import preview_ops_playbook ok, message, data = preview_ops_playbook( { "playbook_key": "onboarding.bootstrap", "node_codes": [normalized_node_code] if normalized_node_code else [], "execution_mode": "control-plane", "auto_approve": True, "requested_by": normalized_requested_by, "payload": bootstrap_plan_request, } ) follow_up = { "next_stage_code": "acceptance_ready", "summary": ( str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() or "接入工单预览已生成;正式执行后应继续观察 Node Agent register / heartbeat,并确认 onboarding_stage 是否进入 acceptance_ready。" ), "recommended_commands": [ build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code), ] if normalized_node_code else [], "focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})), } return ok, message, { "node_code": normalized_node_code, "requested_by": normalized_requested_by, "execution_mode": "control-plane", "bootstrap_plan_request": bootstrap_plan_request, "onboarding": onboarding, "playbook_preview": data, "follow_up": follow_up, } def execute_managed_node_onboarding_bootstrap( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {}) normalized_requested_by = str(requested_by or "api").strip() or "api" from app.services.ops_playbook_service import execute_ops_playbook ok, message, data = execute_ops_playbook( { "playbook_key": "onboarding.bootstrap", "node_codes": [normalized_node_code] if normalized_node_code else [], "execution_mode": "control-plane", "auto_approve": True, "requested_by": normalized_requested_by, "payload": bootstrap_plan_request, } ) follow_up = { "next_stage_code": "acceptance_ready", "summary": ( str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() or "接入工单已签发;接下来应继续观察 Node Agent register / heartbeat,并确认 onboarding_stage 是否进入 acceptance_ready。" ), "recommended_commands": [ build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code), ] if normalized_node_code else [], "focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})), } return ok, message, { "node_code": normalized_node_code, "requested_by": normalized_requested_by, "execution_mode": "control-plane", "bootstrap_plan_request": bootstrap_plan_request, "onboarding": onboarding, "playbook_run": data, "follow_up": follow_up, } def preview_managed_node_onboarding_acceptance( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} acceptance = dict(onboarding.get("acceptance") or {}) acceptance_status_code = str(acceptance.get("status_code") or "").strip() if acceptance_status_code not in {"ready", "ssh_only"}: return False, ( str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收" ), {"onboarding": onboarding} from app.services.ops_playbook_service import preview_ops_playbook normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() normalized_requested_by = str(requested_by or "api").strip() or "api" execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent" ok, message, data = preview_ops_playbook( { "playbook_key": "onboarding.acceptance", "node_codes": [normalized_node_code] if normalized_node_code else [], "execution_mode": execution_mode, "auto_approve": True, "requested_by": normalized_requested_by, } ) follow_up = { "next_stage_code": "ready", "summary": ( str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() or "接管验收预览已生成;正式执行后应继续确认 health / node-agent / worker 三段验收全部通过。" ), "recommended_commands": [ build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"), ] if normalized_node_code else [], "focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})), } return ok, message, { "node_code": normalized_node_code, "requested_by": normalized_requested_by, "execution_mode": execution_mode, "onboarding": onboarding, "acceptance": acceptance, "playbook_preview": data, "follow_up": follow_up, } def execute_managed_node_onboarding_acceptance( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} acceptance = dict(onboarding.get("acceptance") or {}) acceptance_status_code = str(acceptance.get("status_code") or "").strip() if acceptance_status_code not in {"ready", "ssh_only"}: return False, ( str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收" ), {"onboarding": onboarding} from app.services.ops_playbook_service import execute_ops_playbook normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() normalized_requested_by = str(requested_by or "api").strip() or "api" execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent" ok, message, data = execute_ops_playbook( { "playbook_key": "onboarding.acceptance", "node_codes": [normalized_node_code] if normalized_node_code else [], "execution_mode": execution_mode, "auto_approve": True, "requested_by": normalized_requested_by, } ) follow_up = { "next_stage_code": "ready", "summary": ( str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() or "接管验收已发起;接下来应继续确认 health / node-agent / worker 三段验收全部通过。" ), "recommended_commands": [ build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"), ] if normalized_node_code else [], "focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})), } return ok, message, { "node_code": normalized_node_code, "requested_by": normalized_requested_by, "execution_mode": execution_mode, "onboarding": onboarding, "acceptance": acceptance, "playbook_run": data, "follow_up": follow_up, } def preview_managed_node_onboarding_recovery( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} recovery = dict(onboarding.get("recovery_decision") or {}) recovery_action = str(recovery.get("action") or "").strip() if recovery_action == "bootstrap_run": ok, message, data = preview_managed_node_onboarding_bootstrap( node_code=str(onboarding.get("node_code") or node_code or "").strip(), requested_by=requested_by, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) return ok, message, { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, "selected_preview": data, } if recovery_action == "run_acceptance": ok, message, data = preview_managed_node_onboarding_acceptance( node_code=str(onboarding.get("node_code") or node_code or "").strip(), requested_by=requested_by, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) return ok, message, { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, "selected_preview": data, } return False, ( str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作" ), { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, } def execute_managed_node_onboarding_recovery( *, node_code: str, requested_by: str = "api", control_plane_base_url: str = "", root_dir: str = "", participation_payload: dict | None = None, nodes_payload: dict | None = None, ) -> tuple[bool, str, dict]: onboarding = get_managed_node_onboarding( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) if not onboarding: return False, "节点不存在", {} recovery = dict(onboarding.get("recovery_decision") or {}) recovery_action = str(recovery.get("action") or "").strip() if recovery_action == "bootstrap_run": ok, message, data = execute_managed_node_onboarding_bootstrap( node_code=str(onboarding.get("node_code") or node_code or "").strip(), requested_by=requested_by, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) return ok, message, { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, "selected_run": data, } if recovery_action == "run_acceptance": ok, message, data = execute_managed_node_onboarding_acceptance( node_code=str(onboarding.get("node_code") or node_code or "").strip(), requested_by=requested_by, control_plane_base_url=control_plane_base_url, root_dir=root_dir, participation_payload=participation_payload, nodes_payload=nodes_payload, ) return ok, message, { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, "selected_run": data, } return False, ( str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作" ), { "recovery_action": recovery_action, "recovery_decision": recovery, "onboarding": onboarding, } def build_managed_node_handover_bootstrap_plan( *, node_code: str, issued_by: str = "api", expires_in_hours: int = 72, control_plane_base_url: str = "", root_dir: str = "", metadata: dict | None = None, ) -> tuple[bool, str, dict]: handover = get_managed_node_handover( node_code, control_plane_base_url=control_plane_base_url, root_dir=root_dir, ) if not handover: return False, "节点不存在", {} node = dict(handover.get("node") or {}) bootstrap_defaults = dict(handover.get("bootstrap_defaults") or {}) normalized_node_code = str(node.get("node_code") or node_code or "").strip() ok, message, data = build_node_agent_bootstrap_plan( node_code=normalized_node_code, node_region=str( bootstrap_defaults.get("node_region") or node.get("region") or "mainland" ).strip() or "mainland", node_role=str(bootstrap_defaults.get("node_role") or node.get("role") or "worker").strip() or "worker", issued_by=str(issued_by or "api").strip() or "api", expires_in_hours=int(expires_in_hours or bootstrap_defaults.get("expires_in_hours") or 72), control_plane_base_url=str( control_plane_base_url or bootstrap_defaults.get("control_plane_base_url") or handover.get("control_plane_base_url") or "" ).strip(), root_dir=str(root_dir or bootstrap_defaults.get("root_dir") or handover.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck", metadata={ **dict(bootstrap_defaults.get("metadata") or {}), **dict(metadata or {}), "issued_from": str((metadata or {}).get("issued_from") or "ops-managed-node-handover").strip() or "ops-managed-node-handover", }, ) if not ok: return False, message, data refreshed_handover = get_managed_node_handover( normalized_node_code, control_plane_base_url=str(handover.get("control_plane_base_url") or "").strip(), root_dir=str(handover.get("root_dir") or "").strip(), ) return True, message, { **data, "handover": refreshed_handover, "requested_via": "managed-node-handover", } def _agent_job_type(job: dict) -> str: normalized_job = dict(job or {}) metadata = dict(normalized_job.get("metadata") or {}) payload = dict(normalized_job.get("payload") or {}) explicit_job_type = str(metadata.get("job_type") or payload.get("job_type") or "").strip() if explicit_job_type: return explicit_job_type action = str(normalized_job.get("action") or "").strip() rollout_id = int(normalized_job.get("rollout_id") or 0) if rollout_id > 0: return "release_rollout" if action == "deploy.release": return "release_deploy" if action in {"health.snapshot", "logs.collect", "diagnostics.collect"}: return "inspection" return "ops_action" def _agent_release_context(job: dict) -> dict: normalized_job = dict(job or {}) metadata = dict(normalized_job.get("metadata") or {}) payload = dict(normalized_job.get("payload") or {}) return { "release_id": int(payload.get("release_id") or metadata.get("release_id") or 0), "release_version": str(payload.get("release_version") or metadata.get("release_version") or "").strip(), "rollout_id": int( normalized_job.get("rollout_id") or payload.get("rollout_id") or metadata.get("rollout_id") or 0 ), "rollout_code": str(payload.get("rollout_code") or metadata.get("rollout_code") or "").strip(), } def _agent_job_envelope(job: dict) -> dict: normalized_job = dict(job or {}) steps = list(normalized_job.get("steps") or []) first_step = dict(steps[0] or {}) if steps else {} metadata = dict(normalized_job.get("metadata") or {}) focus_ref = dict(normalized_job.get("focus_ref") or {}) if not focus_ref: focus_ref = { "kind": "ops_job", "job_id": int(normalized_job.get("id") or 0), "job_code": str(normalized_job.get("job_code") or "").strip(), "action": str(normalized_job.get("action") or "").strip(), "target_node_code": str(normalized_job.get("target_node_code") or "").strip(), } step_key = str(first_step.get("step_key") or metadata.get("step_key") or "dispatch").strip() or "dispatch" step_title = ( str( first_step.get("title") or metadata.get("step_title") or normalized_job.get("summary") or normalized_job.get("action") or "" ).strip() or step_key ) envelope = { **normalized_job, "protocol_version": "ops-agent/v1", "envelope_type": "agent_job", "job_id": int(normalized_job.get("id") or 0), "job_type": _agent_job_type(normalized_job), "step_key": step_key, "step_title": step_title, "policy": dict(normalized_job.get("policy") or {}), "release_context": _agent_release_context(normalized_job), "focus_ref": focus_ref, "job_ref": { "job_id": int(normalized_job.get("id") or 0), "job_code": str(normalized_job.get("job_code") or "").strip(), "action": str(normalized_job.get("action") or "").strip(), "target_node_code": str(normalized_job.get("target_node_code") or "").strip(), }, "step_ref": { "step_id": int(first_step.get("id") or 0), "step_key": step_key, "step_title": step_title, }, } return envelope def _completion_summary(job: dict, *, client_request_id: str = "", deduplicated: bool = False) -> dict: normalized_job = dict(job or {}) result = dict(normalized_job.get("result") or {}) return { "job_id": int(normalized_job.get("id") or 0), "job_code": str(normalized_job.get("job_code") or "").strip(), "status": str(normalized_job.get("status") or "").strip(), "status_label": str(normalized_job.get("status_label") or "").strip(), "deduplicated": bool(deduplicated), "client_request_id": str(client_request_id or "").strip(), "result_summary_text": str(result.get("summary_text") or result.get("summary") or "").strip(), "focus_ref": dict(normalized_job.get("focus_ref") or {}), } def _slim_delivery_queue_head_record(node: dict, *, state: str) -> dict: normalized_state = str(state or "").strip() if normalized_state not in {"pending", "dead_letter"}: return {} if normalized_state == "pending": created_at = str(node.get("delivery_queue_oldest_pending_at") or "").strip() request_id = str(node.get("delivery_queue_oldest_pending_request_id") or "").strip() request_kind = str(node.get("delivery_queue_oldest_pending_kind") or "").strip() else: created_at = str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip() request_id = str(node.get("delivery_queue_oldest_dead_letter_request_id") or "").strip() request_kind = str(node.get("delivery_queue_oldest_dead_letter_kind") or "").strip() if not any((created_at, request_id, request_kind)): return {} node_code = str(node.get("node_code") or "").strip() return { "record_key": f"{node_code}:{normalized_state}:{request_id or request_kind or created_at or 'head'}", "record_id": request_id, "node_code": node_code, "state": normalized_state, "label": "待重试头部记录" if normalized_state == "pending" else "死信头部记录", "request_kind": request_kind, "client_request_id": request_id, "created_at": created_at if normalized_state == "pending" else "", "dead_letter_at": created_at if normalized_state == "dead_letter" else "", "detail_code": "", "error_message": "", "record_source": "agent-heartbeat-snapshot", "record_visibility": "head_only", "payload_visibility": "unavailable", "response_visibility": "unavailable", } def get_managed_node_delivery_queue(node_code: str) -> dict: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return {} payload = list_managed_nodes_with_agent_state() node = next( ( dict(item or {}) for item in list(payload.get("nodes") or []) if str(item.get("node_code") or "").strip() == normalized_node_code ), {}, ) if not node: return {} pending_head = _slim_delivery_queue_head_record(node, state="pending") dead_letter_head = _slim_delivery_queue_head_record(node, state="dead_letter") record_visibility = "head_only" capabilities = { "can_flush": True, "can_replay": True, "can_discard": True, } return { "node_code": normalized_node_code, "node": { "node_code": normalized_node_code, "title": str(node.get("title") or "").strip(), "region": str(node.get("region") or "").strip(), "role": str(node.get("role") or "").strip(), "is_managed": bool(node.get("is_managed", False)), "is_enabled": bool(node.get("is_enabled", False)), "is_agent_online": bool(node.get("is_agent_online", False)), "agent_state": str(node.get("agent_state") or "").strip(), "agent_state_label": str(node.get("agent_state_label") or "").strip(), "remote_access_state": str(node.get("remote_access_state") or "").strip(), "remote_access_label": str(node.get("remote_access_label") or "").strip(), }, "summary": { "state": str(node.get("delivery_queue_state") or "").strip(), "label": str(node.get("delivery_queue_label") or "").strip(), "reason": str(node.get("delivery_queue_reason") or "").strip(), "pending_count": int(node.get("delivery_queue_pending_count", 0) or 0), "dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0), "last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(), "oldest_pending_at": str(node.get("delivery_queue_oldest_pending_at") or "").strip(), "oldest_dead_letter_at": str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip(), }, "record_visibility": record_visibility, "capabilities": capabilities, "records_capability": { "record_source": "agent-heartbeat-snapshot", "record_visibility": record_visibility, "full_records_available": False, "available_actions": [ { "key": "delivery.queue.flush", "label": "立即冲刷", "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/flush", "record_scope": "pending", "execution_mode": "remote-agent", }, { "key": "delivery.queue.replay", "label": "重放死信", "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/replay", "record_scope": "dead_letter", "execution_mode": "remote-agent", }, { "key": "delivery.queue.discard", "label": "丢弃死信", "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/records/{{record_id}}/discard", "record_scope": "dead_letter", "execution_mode": "remote-agent", }, ], "next_step": "当前控制面仍以头部记录视角展示队列;已支持通过正式 Ops Job 触发冲刷、重放和丢弃动作。", }, "head_records": { "pending": pending_head, "dead_letter": dead_letter_head, }, } def list_managed_node_delivery_queue_records( node_code: str, *, state: str = "", limit: int = 50, ) -> dict: normalized_node_code = str(node_code or "").strip() normalized_state = str(state or "").strip() safe_limit = min(max(int(limit or 50), 1), 200) queue_payload = get_managed_node_delivery_queue(normalized_node_code) if not queue_payload: return {} candidate_records = [ record for record in ( dict((queue_payload.get("head_records") or {}).get("pending") or {}), dict((queue_payload.get("head_records") or {}).get("dead_letter") or {}), ) if record ] if normalized_state: candidate_records = [ record for record in candidate_records if str(record.get("state") or "").strip() == normalized_state ] records = candidate_records[:safe_limit] available_state_counts: dict[str, int] = {} for record in candidate_records: record_state = str(record.get("state") or "").strip() or "unknown" available_state_counts[record_state] = int(available_state_counts.get(record_state, 0) or 0) + 1 return { "node_code": normalized_node_code, "records": records, "summary": { "total": len(records), "returned_records": len(records), "available_total": len(candidate_records), "available_state_counts": available_state_counts, "record_visibility": "head_only", "filters": { "state": normalized_state, "limit": safe_limit, }, }, "records_capability": dict(queue_payload.get("records_capability") or {}), "record_visibility": str((queue_payload.get("record_visibility") or "head_only")).strip() or "head_only", "capabilities": dict(queue_payload.get("capabilities") or {}), } def _delivery_queue_visible_record_id(record: dict | None = None) -> str: normalized_record = dict(record or {}) return ( str(normalized_record.get("record_id") or "").strip() or str(normalized_record.get("client_request_id") or "").strip() or str(normalized_record.get("record_key") or "").strip() ) def _find_visible_delivery_queue_record(queue_payload: dict, record_id: str) -> dict: normalized_record_id = str(record_id or "").strip() if not normalized_record_id: return {} for state in ("pending", "dead_letter"): record = dict(((queue_payload.get("head_records") or {}).get(state) or {})) if record and _delivery_queue_visible_record_id(record) == normalized_record_id: return record return {} def _build_delivery_queue_selector_from_template_payload( template_key: str, normalized_payload: dict, *, force_state: str = "", ) -> dict: selector: dict[str, object] = {} if str(force_state or "").strip(): selector["state"] = str(force_state or "").strip() elif template_key in {"delivery.queue.replay", "delivery.queue.discard"}: selector["state"] = "dead_letter" record_id = str(normalized_payload.get("record_id") or "").strip() request_kind = str(normalized_payload.get("request_kind") or "").strip() detail_code = str(normalized_payload.get("detail_code") or "").strip() if record_id: selector["record_id"] = record_id if request_kind: selector["request_kind"] = request_kind if detail_code: selector["detail_code"] = detail_code return selector def request_managed_node_delivery_queue_action( node_code: str, action_key: str, *, payload: dict | None = None, record_id: str = "", ) -> tuple[bool, str, dict]: normalized_node_code = str(node_code or "").strip() normalized_action_key = str(action_key or "").strip() raw_payload = dict(payload or {}) if not normalized_node_code: return False, "node_code 不能为空", {} if normalized_action_key not in {"delivery.queue.flush", "delivery.queue.replay", "delivery.queue.discard"}: return False, "当前不支持该 delivery queue 动作", {"action_key": normalized_action_key} queue_payload = get_managed_node_delivery_queue(normalized_node_code) if not queue_payload: return False, "节点不存在", {} template = get_ops_action_template(normalized_action_key) if not template: return False, "delivery queue 动作模板不存在", {"action_key": normalized_action_key} effective_payload = dict(raw_payload) if str(record_id or "").strip(): effective_payload["record_id"] = str(record_id or "").strip() normalize_ok, normalize_message, normalized_action_payload = build_ops_template_payload( normalized_action_key, effective_payload, ) if not normalize_ok: return False, normalize_message, {"action_key": normalized_action_key} visible_record = {} if str(record_id or "").strip(): visible_record = _find_visible_delivery_queue_record(queue_payload, str(record_id or "").strip()) if not visible_record: return False, "当前控制面仅支持对可见头部记录执行单条动作", { "action_key": normalized_action_key, "node_code": normalized_node_code, "record_id": str(record_id or "").strip(), "record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")), } if normalized_action_key in {"delivery.queue.replay", "delivery.queue.discard"}: if str(visible_record.get("state") or "").strip() != "dead_letter": return False, "单条重放/丢弃当前仅支持死信头部记录", { "action_key": normalized_action_key, "node_code": normalized_node_code, "record": visible_record, } normalized_action_payload["record_id"] = _delivery_queue_visible_record_id(visible_record) selector = _build_delivery_queue_selector_from_template_payload( normalized_action_key, normalized_action_payload, ) action_payload: dict[str, object] = {} if normalized_action_key == "delivery.queue.flush": action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) elif normalized_action_key == "delivery.queue.replay": action_payload["selector"] = selector action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) action_payload["flush_after_replay"] = bool(normalized_action_payload.get("flush_after_replay", True)) if str(normalized_action_payload.get("reason") or "").strip(): action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip() elif normalized_action_key == "delivery.queue.discard": action_payload["selector"] = selector action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) action_payload["discarded_by"] = ( str(raw_payload.get("discarded_by") or raw_payload.get("requested_by") or "web-ui").strip() or "web-ui" ) action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip() requested_by = str(raw_payload.get("requested_by") or "web-ui").strip() or "web-ui" ok, message, data = create_ops_job( { "action": normalized_action_key, "target_type": "node", "target_node_code": normalized_node_code, "requested_by": requested_by, "execution_mode": "remote-agent", "auto_approve": bool(template.get("default_auto_approve", False)), "payload": action_payload, "metadata": { "delivery_queue_action": { "action_key": normalized_action_key, "template_key": normalized_action_key, "selector": selector, "record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")), "requested_from": "ops-delivery-queue", "visible_record": visible_record, "queue_summary": dict(queue_payload.get("summary") or {}), } }, } ) if not ok: return False, message, data action_label = str(template.get("title") or normalized_action_key).strip() return True, f"{action_label}任务已创建", { "node_code": normalized_node_code, "action_key": normalized_action_key, "action_label": action_label, "queue": { "summary": dict(queue_payload.get("summary") or {}), "records_capability": dict(queue_payload.get("records_capability") or {}), }, "selector": selector, "visible_record": visible_record, "job": dict(data.get("job") or {}), "executed_immediately": bool(data.get("executed_immediately", False)), } def issue_node_agent_token( *, node_code: str, issued_by: str = "api", expires_in_hours: int = 72, metadata: dict | None = None, ) -> tuple[bool, str, dict]: ensure_ops_agent_schema() normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return False, "node_code 不能为空", {} token = secrets.token_urlsafe(32) token_hash = _hash_token(token) expires_at = datetime.now() + timedelta(hours=max(1, min(int(expires_in_hours or 72), 24 * 30))) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO ops_node_tokens ( token_hash, node_code, purpose, issued_by, is_enabled, expires_at, metadata_json, created_at, updated_at ) VALUES (%s, %s, 'agent', %s, TRUE, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id, expires_at, created_at """, ( token_hash, normalized_node_code, str(issued_by or "api").strip() or "api", expires_at, json.dumps(metadata or {}, ensure_ascii=False), ), ) row = cur.fetchone() conn.commit() return True, "节点 Agent 令牌已签发", { "token": token, "token_preview": f"{token[:6]}...{token[-4:]}", "record_id": int(row[0]), "node_code": normalized_node_code, "expires_at": _format_time(row[1]), "created_at": _format_time(row[2]), } def _normalize_control_plane_base_url(raw_value: str) -> str: normalized = str(raw_value or "").strip().rstrip("/") if normalized.endswith("/api/v1"): normalized = normalized[: -len("/api/v1")] return normalized def _candidate_node_agent_install_paths(root_dir: str) -> list[str]: normalized_root_dir = str(root_dir or "/opt/domaincheck").strip().rstrip("/") or "/opt/domaincheck" return [ f"{normalized_root_dir}/domainCheck/deploy/multi-region/bootstrap_node_agent.sh", f"{normalized_root_dir}/domain-api/deploy/multi-region/bootstrap_node_agent.sh", f"{normalized_root_dir}/deploy/multi-region/bootstrap_node_agent.sh", ] def _build_node_agent_install_block(root_dir: str) -> tuple[str, str]: candidates = _candidate_node_agent_install_paths(root_dir) preferred_path = candidates[0] lines = [ f"ROOT_DIR={shlex.quote(root_dir)}", "INSTALL_SCRIPT=", "for candidate in \\", ] for index, candidate in enumerate(candidates): suffix = " \\" if index < len(candidates) - 1 else "" lines.append(f" {shlex.quote(candidate)}{suffix}") lines.extend( [ "do", " if [[ -f \"${candidate}\" ]]; then", " INSTALL_SCRIPT=\"${candidate}\"", " break", " fi", "done", "", "if [[ -z \"${INSTALL_SCRIPT}\" ]]; then", " echo \"bootstrap_node_agent.sh not found under known layouts\" >&2", " exit 1", "fi", "", "bash \"${INSTALL_SCRIPT}\" \"${ROOT_DIR}\"", ] ) return preferred_path, "\n".join(lines) def _build_node_agent_bootstrap_script( *, node_code: str, root_dir: str, install_script_path: str, install_command_block: str, env_content: str, health_checks: list[str], ) -> tuple[str, str, str]: script_name = f"bootstrap-node-agent-{node_code}.sh" script_path = f"/tmp/{script_name}" script_lines = [ "#!/usr/bin/env bash", "set -euo pipefail", "", f"ROOT_DIR={shlex.quote(root_dir)}", f"INSTALL_SCRIPT={shlex.quote(install_script_path)}", "ENV_FILE=/etc/default/domaincheck-node-agent", "SERVICE_NAME=domaincheck-node-agent", "", "# Resolve bootstrap_node_agent.sh across supported repo layouts.", *install_command_block.splitlines(), "", "cat >\"${ENV_FILE}\" <<'EOF'", env_content, "EOF", "", "systemctl restart \"${SERVICE_NAME}\"", "systemctl status \"${SERVICE_NAME}\" --no-pager -l", "journalctl -u \"${SERVICE_NAME}\" -n 60 --no-pager", ] if health_checks: script_lines.extend( [ "", "echo", "echo '[health-checks]'", ] ) for item in health_checks: normalized_item = str(item or "").strip() if normalized_item: script_lines.append(normalized_item) script_content = "\n".join(script_lines).strip() + "\n" write_script_block = "\n".join( [ f"cat >{script_path} <<'EOF'", script_content.rstrip("\n"), "EOF", f"chmod +x {script_path}", ] ) run_script_block = "\n".join( [ write_script_block, f"bash {script_path}", ] ) return script_name, script_content, run_script_block def build_node_agent_bootstrap_plan( *, node_code: str, node_region: str = "mainland", node_role: str = "worker", issued_by: str = "api", expires_in_hours: int = 72, control_plane_base_url: str = "", root_dir: str = "/opt/domaincheck", metadata: dict | None = None, ) -> tuple[bool, str, dict]: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return False, "node_code 不能为空", {} normalized_region = str(node_region or "mainland").strip() or "mainland" normalized_role = str(node_role or "worker").strip() or "worker" normalized_root_dir = str(root_dir or "/opt/domaincheck").strip() or "/opt/domaincheck" normalized_control_plane_base_url = _normalize_control_plane_base_url(control_plane_base_url) if not normalized_control_plane_base_url: normalized_control_plane_base_url = f"http://127.0.0.1:{settings.api_port}" token_ok, token_message, token_data = issue_node_agent_token( node_code=normalized_node_code, issued_by=issued_by, expires_in_hours=expires_in_hours, metadata=metadata or {}, ) if not token_ok: return False, token_message, {} env_lines = [ f"OPS_CONTROL_PLANE_BASE_URL={normalized_control_plane_base_url}", f"OPS_AGENT_TOKEN={token_data.get('token') or ''}", "OPS_AGENT_POLL_INTERVAL_SECONDS=5", "", f"NODE_CODE={normalized_node_code}", f"NODE_REGION={normalized_region}", f"NODE_ROLE={normalized_role}", "", f"WORKER_SERVICE_NAME={settings.worker_service_name}", f"API_SERVICE_NAME={settings.api_service_name}", f"SYNC_AGENT_SERVICE_NAME={settings.sync_agent_service_name}", "NODE_AGENT_SERVICE_NAME=domaincheck-node-agent", "", 'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.start_detection","runtime.stop_detection","runtime.pull_tasks","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","runtime.reset_lab_state","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]', "OPS_AGENT_LABELS={}", ] env_content = "\n".join(env_lines) install_script_path, install_command_block = _build_node_agent_install_block(normalized_root_dir) command_lines = [ install_command_block, "cat >/etc/default/domaincheck-node-agent <<'EOF'", env_content, "EOF", "systemctl restart domaincheck-node-agent", "systemctl status domaincheck-node-agent --no-pager -l", "journalctl -u domaincheck-node-agent -n 60 --no-pager", ] command_block = "\n".join(command_lines) health_checks = [ "systemctl status domaincheck-node-agent --no-pager -l", "journalctl -u domaincheck-node-agent -n 60 --no-pager", f"curl -s {normalized_control_plane_base_url}/api/v1/ops/overview", ] bootstrap_script_name, bootstrap_script_content, bootstrap_run_script_block = _build_node_agent_bootstrap_script( node_code=normalized_node_code, root_dir=normalized_root_dir, install_script_path=install_script_path, install_command_block=install_command_block, env_content=env_content, health_checks=health_checks, ) return True, "节点 Agent 接入方案已生成", { **token_data, "control_plane_base_url": normalized_control_plane_base_url, "bootstrap_plan": { "node_code": normalized_node_code, "node_region": normalized_region, "node_role": normalized_role, "root_dir": normalized_root_dir, "env_file": "/etc/default/domaincheck-node-agent", "service_name": "domaincheck-node-agent", "service_file": "/etc/systemd/system/domaincheck-node-agent.service", "install_script_path": install_script_path, "install_script_candidates": _candidate_node_agent_install_paths(normalized_root_dir), "install_command_block": install_command_block, "env_content": env_content, "command_lines": command_lines, "command_block": command_block, "health_checks": health_checks, "health_check_block": "\n".join(health_checks), "bootstrap_script_name": bootstrap_script_name, "bootstrap_script_path": f"/tmp/{bootstrap_script_name}", "bootstrap_script_content": bootstrap_script_content, "bootstrap_run_script_block": bootstrap_run_script_block, }, } def _authenticate_agent_token(token: str, *, expected_node_code: str | None = None) -> tuple[bool, str, dict]: ensure_ops_agent_schema() token_hash = _hash_token(token) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, node_code, purpose, is_enabled, expires_at, metadata_json FROM ops_node_tokens WHERE token_hash = %s LIMIT 1 """, (token_hash,), ) row = cur.fetchone() if not row: return _agent_error("agent token 无效", "agent_token_invalid") record_id = int(row[0]) node_code = str(row[1] or "") purpose = str(row[2] or "") is_enabled = bool(row[3]) expires_at = row[4] metadata = _decode_json(row[5]) if purpose != "agent" or not is_enabled: return _agent_error("agent token 不可用", "agent_token_unavailable") if isinstance(expires_at, datetime): current_time = datetime.now(expires_at.tzinfo) if expires_at.tzinfo else datetime.now() if expires_at < current_time: return _agent_error("agent token 已过期", "agent_token_expired") normalized_expected = str(expected_node_code or "").strip() if normalized_expected and normalized_expected != node_code: return _agent_error( "agent token 与节点不匹配", "agent_token_node_mismatch", {"expected_node_code": normalized_expected, "token_node_code": node_code}, ) cur.execute( """ UPDATE ops_node_tokens SET last_used_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (record_id,), ) conn.commit() return True, "ok", { "record_id": record_id, "node_code": node_code, "metadata": metadata, "expires_at": _format_time(expires_at), } def _upsert_agent_runtime(node_code: str, payload: dict) -> None: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return region = str(payload.get("region") or "unknown").strip() or "unknown" role = str(payload.get("role") or "worker").strip() or "worker" title = str(payload.get("title") or normalized_node_code).strip() or normalized_node_code metadata = payload.get("metadata") or {} ssh_host = str(payload.get("ssh_host") or "").strip() ssh_user = str(payload.get("ssh_user") or "").strip() ssh_port = max(1, int(payload.get("ssh_port") or 22)) auth_mode = str(payload.get("auth_mode") or "key").strip() or "key" deploy_channel = str(payload.get("deploy_channel") or "stable").strip() or "stable" merged_metadata = { **metadata, "agent_version": str(payload.get("agent_version") or metadata.get("agent_version") or "").strip(), "last_seen_at": datetime.now().isoformat(timespec="seconds"), "capabilities": payload.get("capabilities") or metadata.get("capabilities") or [], "labels": payload.get("labels") or metadata.get("labels") or {}, "hostname": str(payload.get("hostname") or metadata.get("hostname") or "").strip(), "ip": str(payload.get("ip") or metadata.get("ip") or "").strip(), } with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO ops_managed_nodes ( node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, TRUE, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ON CONFLICT (node_code) DO UPDATE SET region = EXCLUDED.region, role = EXCLUDED.role, title = EXCLUDED.title, ssh_host = CASE WHEN EXCLUDED.ssh_host <> '' THEN EXCLUDED.ssh_host ELSE ops_managed_nodes.ssh_host END, ssh_port = EXCLUDED.ssh_port, ssh_user = CASE WHEN EXCLUDED.ssh_user <> '' THEN EXCLUDED.ssh_user ELSE ops_managed_nodes.ssh_user END, auth_mode = EXCLUDED.auth_mode, deploy_channel = EXCLUDED.deploy_channel, metadata_json = EXCLUDED.metadata_json, last_seen_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP """, ( normalized_node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, json.dumps(merged_metadata, ensure_ascii=False), ), ) conn.commit() def _load_existing_detect_node_runtime(node_code: str) -> dict: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return {} try: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT status, current_load, metadata_json, last_heartbeat_at FROM detect_worker_nodes WHERE node_code = %s LIMIT 1 """, (normalized_node_code,), ) row = cur.fetchone() except Exception: return {} if not row: return {} metadata = row[2] if isinstance(row[2], dict) else {} return { "status": str(row[0] or "").strip(), "current_load": int(row[1] or 0), "metadata": metadata, "last_heartbeat_at": row[3], } def _upsert_agent_detect_runtime(node_code: str, payload: dict) -> None: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {} detect_runtime = metadata.get("detect_runtime") if isinstance(metadata.get("detect_runtime"), dict) else {} if not detect_runtime: return try: from app.services.cluster_runtime_service import register_node_heartbeat region = str(payload.get("region") or "unknown").strip() or "unknown" role = str(payload.get("role") or "worker").strip() or "worker" active_threads = max(0, int(detect_runtime.get("active_threads") or 0)) max_threads = max(0, int(detect_runtime.get("max_threads") or 0)) current_load = max( 0, int(detect_runtime.get("current_load") or 0), active_threads, ) worker_online = bool(detect_runtime.get("worker_online", False) or detect_runtime.get("service_running", False)) detect_participating = bool(detect_runtime.get("detect_participating", False) or current_load > 0) phase_label = str(detect_runtime.get("phase_label") or "").strip() phase_detail = str(detect_runtime.get("phase_detail") or "").strip() recent_warning = str(detect_runtime.get("recent_warning") or "").strip() has_runtime_signal = any( [ worker_online, detect_participating, current_load > 0, active_threads > 0, max_threads > 0, bool(phase_label), bool(phase_detail), bool(recent_warning), ] ) if not has_runtime_signal: return existing_runtime = _load_existing_detect_node_runtime(normalized_node_code) existing_metadata = existing_runtime.get("metadata") if isinstance(existing_runtime.get("metadata"), dict) else {} existing_active_threads = max(0, int(existing_metadata.get("active_threads", 0) or 0)) existing_current_load = max(0, int(existing_runtime.get("current_load", 0) or 0), existing_active_threads) existing_status = str(existing_runtime.get("status") or "").strip() existing_last_heartbeat_at = existing_runtime.get("last_heartbeat_at") existing_is_recent = False if isinstance(existing_last_heartbeat_at, datetime): current_time = ( datetime.now(existing_last_heartbeat_at.tzinfo) if existing_last_heartbeat_at.tzinfo else datetime.now() ) existing_is_recent = (current_time - existing_last_heartbeat_at) <= timedelta(seconds=120) normalized_phase_detail = phase_detail.lower() generic_phase_detail = bool(normalized_phase_detail) and "/" in normalized_phase_detail and " " not in normalized_phase_detail weak_agent_snapshot = bool( worker_online and current_load <= 0 and active_threads <= 0 and max_threads > 0 and not phase_label and (not phase_detail or generic_phase_detail) and not recent_warning ) if weak_agent_snapshot and existing_is_recent and ( existing_current_load > 0 or existing_status == "busy" ): return status = "busy" if current_load > 0 else ("online" if worker_online else "unknown") heartbeat_metadata = { "service": "agent-heartbeat", "worker_online": worker_online, "detect_participating": detect_participating, "active_threads": active_threads, "max_threads": max_threads, "phase_label": phase_label, "phase_detail": phase_detail, "recent_warning": recent_warning, "updated_at": str(detect_runtime.get("updated_at") or "").strip(), "agent_heartbeat_at": datetime.now().isoformat(timespec="seconds"), } register_node_heartbeat( node_code=normalized_node_code, region=region, role=role, status=status, current_load=current_load, metadata=heartbeat_metadata, hostname_override=str(payload.get("hostname") or "").strip(), ip_override=str(payload.get("ip") or "").strip(), ) except Exception: return def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, auth _upsert_agent_runtime(node_code, payload) _upsert_agent_detect_runtime(node_code, payload) return True, "Agent 注册成功", { "node_code": node_code, "expires_at": auth.get("expires_at", ""), "capabilities": payload.get("capabilities") or [], } def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, auth _upsert_agent_runtime(node_code, payload) _upsert_agent_detect_runtime(node_code, payload) return True, "heartbeat ok", { "node_code": node_code, "server_time": _format_time(datetime.now()), "expires_at": auth.get("expires_at", ""), } def _runtime_config_bundle_hash_payload(bundle: dict | None) -> dict: normalized = dict(bundle or {}) normalized.pop("config_hash", None) normalized.pop("generated_at", None) return normalized def _build_agent_runtime_config_bundle(node_code: str) -> dict: settings_payload = get_settings_payload() runtime_settings = get_runtime_settings() sensitive_words_payload = get_sensitive_words_payload() bundle = { "node_code": str(node_code or "").strip(), "detect_options": dict(settings_payload.get("detect_options") or {}), "proxy_config": dict(settings_payload.get("proxy_config") or {}), "thread_count": int(settings_payload.get("thread_count", 2) or 2), "node_thread_counts": dict(settings_payload.get("node_thread_counts") or {}), "runtime_settings": dict(runtime_settings or {}), "sensitive_words": { "text": str(sensitive_words_payload.get("text") or ""), "total": int(sensitive_words_payload.get("total", 0) or 0), "items": list(sensitive_words_payload.get("items") or []), }, "generated_at": _format_time(datetime.now()), } bundle["config_hash"] = hashlib.sha256( json.dumps( _runtime_config_bundle_hash_payload(bundle), ensure_ascii=False, sort_keys=True, ).encode("utf-8") ).hexdigest() return bundle def agent_pull_runtime_config(payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, auth bundle = _build_agent_runtime_config_bundle(node_code) return True, "runtime config ok", { "node_code": node_code, "server_time": _format_time(datetime.now()), "expires_at": auth.get("expires_at", ""), "bundle": bundle, } def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, _auth safe_limit = min(max(int(limit or 1), 1), 10) dispatched_events: list[dict] = [] with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute( """ SELECT id FROM ops_jobs WHERE target_node_code = %s AND status = 'queued' AND execution_mode = 'remote-agent' ORDER BY created_at ASC, id ASC LIMIT %s FOR UPDATE SKIP LOCKED """, (node_code, safe_limit), ) rows = cur.fetchall() job_ids = [int(row[0]) for row in rows] jobs: list[dict] = [] for job_id in job_ids: cur.execute( """ UPDATE ops_jobs SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), dispatched_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (job_id,), ) cur.execute( """ UPDATE ops_job_steps SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP WHERE job_id = %s """, (job_id,), ) dispatched_events.append( { "job_id": job_id, "node_code": node_code, "event_type": "agent_dispatched", "message": f"任务已派发给节点 {node_code}", "payload": {"node_code": node_code}, } ) jobs.append(_agent_job_envelope(get_ops_job(job_id))) conn.commit() for event in dispatched_events: append_ops_job_event(**event) return True, "ok", { "jobs": jobs, "count": len(jobs), "node_code": node_code, "limit": safe_limit, "protocol_version": "ops-agent/v1", "envelope_type": "agent_job", } def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, _auth started_event: dict | None = None with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute( """ UPDATE ops_jobs SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP WHERE id = %s AND target_node_code = %s RETURNING id """, (int(job_id), node_code), ) row = cur.fetchone() if not row: conn.rollback() return _agent_error( "任务不存在或不属于当前节点", "ops_job_not_owned_by_agent", {"job_id": int(job_id), "node_code": node_code}, ) cur.execute( """ UPDATE ops_job_steps SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP WHERE job_id = %s RETURNING id """, (int(job_id),), ) step_rows = cur.fetchall() step_ids = [int(item[0]) for item in step_rows] started_event = { "job_id": int(job_id), "node_code": node_code, "event_type": "agent_started", "message": f"节点 {node_code} 已开始执行任务", "payload": {"step_ids": step_ids}, } conn.commit() if started_event: append_ops_job_event(**started_event) job = get_ops_job(int(job_id)) return True, "任务已标记为运行中", { "job": job, "accepted_job": _agent_job_envelope(job), "focus_ref": dict(job.get("focus_ref") or {}), } def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, _auth job_status = str(payload.get("status") or "").strip().lower() if job_status not in {"success", "failed", "partially_succeeded"}: return _agent_error( "status 必须是 success / failed / partially_succeeded", "ops_job_invalid_status", {"allowed_statuses": ["success", "failed", "partially_succeeded"]}, ) stdout_text = str(payload.get("stdout") or "") stderr_text = str(payload.get("stderr") or "") result = dict(payload.get("result") or {}) error_message = str(payload.get("error_message") or "").strip() client_request_id = str(payload.get("client_request_id") or "").strip()[:128] duration_ms = max(0, int(payload.get("duration_ms") or 0)) summary_text = str(payload.get("summary_text") or "").strip() focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} if duration_ms and "duration_ms" not in result: result["duration_ms"] = duration_ms if summary_text and "summary_text" not in result and "summary" not in result: result["summary_text"] = summary_text if focus_ref and "focus_ref" not in result: result["focus_ref"] = focus_ref event_level = "info" if job_status == "success" else ("warning" if job_status == "partially_succeeded" else "error") completed_event: dict | None = None with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute( """ SELECT id, COALESCE(last_agent_complete_request_id, '') FROM ops_jobs WHERE id = %s AND target_node_code = %s LIMIT 1 """, (int(job_id), node_code), ) job_row = cur.fetchone() if not job_row: conn.rollback() return _agent_error( "任务不存在或不属于当前节点", "ops_job_not_owned_by_agent", {"job_id": int(job_id), "node_code": node_code}, ) current_request_id = str(job_row[1] or "") if client_request_id and current_request_id == client_request_id: conn.rollback() deduplicated_job = get_ops_job(int(job_id)) return True, "任务结果已幂等回写", { "job": deduplicated_job, "deduplicated": True, "client_request_id": client_request_id, "completion_summary": _completion_summary( deduplicated_job, client_request_id=client_request_id, deduplicated=True, ), } cur.execute( """ UPDATE ops_jobs SET status = %s, result_json = %s, error_message = %s, last_agent_complete_request_id = CASE WHEN %s <> '' THEN %s ELSE last_agent_complete_request_id END, finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, ( job_status, json.dumps(result, ensure_ascii=False), error_message if job_status != "success" else "", client_request_id, client_request_id, int(job_id), ), ) cur.execute( """ UPDATE ops_job_steps SET status = %s, stdout_text = %s, stderr_text = %s, result_json = %s, finished_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE job_id = %s RETURNING id """, ( job_status, stdout_text, stderr_text, json.dumps(result, ensure_ascii=False), int(job_id), ), ) step_rows = cur.fetchall() step_ids = [int(item[0]) for item in step_rows] completed_event = { "job_id": int(job_id), "node_code": node_code, "client_event_id": (f"complete:{client_request_id}" if client_request_id else ""), "event_type": "agent_completed", "message": f"节点 {node_code} 已完成任务,状态: {job_status}", "level": event_level, "payload": { "step_ids": step_ids, "result": result, "duration_ms": duration_ms, "summary_text": str(result.get("summary_text") or result.get("summary") or summary_text).strip(), "focus_ref": focus_ref, }, } conn.commit() if completed_event: append_ops_job_event(**completed_event) from app.services.ops_release_service import refresh_release_rollout_for_job refresh_release_rollout_for_job(int(job_id)) job = get_ops_job(int(job_id)) return True, "任务结果已回写", { "job": job, "deduplicated": False, "client_request_id": client_request_id, "completion_summary": _completion_summary(job, client_request_id=client_request_id), } def agent_append_job_event(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: node_code = str(payload.get("node_code") or "").strip() if not node_code: return _agent_error("node_code 不能为空", "agent_node_code_required") ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) if not ok: return False, message, _auth client_event_id = str(payload.get("client_event_id") or "").strip()[:128] event_payload = dict(payload.get("payload") or {}) occurred_at = str(payload.get("occurred_at") or "").strip() summary_text = str(payload.get("summary_text") or "").strip() focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} if occurred_at and "occurred_at" not in event_payload: event_payload["occurred_at"] = occurred_at if summary_text and "summary_text" not in event_payload: event_payload["summary_text"] = summary_text if focus_ref and "focus_ref" not in event_payload: event_payload["focus_ref"] = focus_ref event_id = append_ops_job_event( job_id=int(job_id), step_id=int(payload.get("step_id") or 0) or None, node_code=node_code, client_event_id=client_event_id, event_type=str(payload.get("event_type") or "agent_event").strip() or "agent_event", level=str(payload.get("level") or "info").strip() or "info", message=str(payload.get("message") or "").strip()[:2000] or "agent event", payload=event_payload, ) return True, "事件已写入", { "event_id": event_id, "client_event_id": client_event_id, "event": get_ops_job_event(event_id), }