995 lines
36 KiB
Python
995 lines
36 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import socket
|
||
import subprocess
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from datetime import datetime
|
||
from uuid import uuid4
|
||
|
||
from app.services.ops_action_executor_core import (
|
||
build_service_name_map,
|
||
execute_structured_action,
|
||
supports_structured_action,
|
||
)
|
||
from app.services.ops_release_executor_core import (
|
||
execute_release_action,
|
||
normalize_release_health_check_services as _release_normalize_health_check_services,
|
||
normalize_text_list as _release_normalize_text_list,
|
||
)
|
||
|
||
|
||
CONTROL_PLANE_BASE_URL = str(os.getenv("OPS_CONTROL_PLANE_BASE_URL", "")).strip().rstrip("/")
|
||
AGENT_TOKEN = str(os.getenv("OPS_AGENT_TOKEN", "")).strip()
|
||
NODE_CODE = str(os.getenv("NODE_CODE", "")).strip()
|
||
NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland"
|
||
NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker"
|
||
AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5))
|
||
WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker"
|
||
API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api"
|
||
SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent"
|
||
NODE_AGENT_SERVICE_NAME = str(os.getenv("NODE_AGENT_SERVICE_NAME", "domaincheck-node-agent")).strip() or "domaincheck-node-agent"
|
||
AGENT_VERSION = "0.1.0"
|
||
_APP_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
_PROJECT_DIR = os.path.dirname(_APP_DIR)
|
||
_DEFAULT_QUEUE_DIR = os.path.join(_PROJECT_DIR, "runtime", "node-agent-queue", NODE_CODE or "unbound")
|
||
AGENT_QUEUE_DIR = str(os.getenv("OPS_AGENT_QUEUE_DIR", _DEFAULT_QUEUE_DIR)).strip() or _DEFAULT_QUEUE_DIR
|
||
AGENT_QUEUE_FLUSH_LIMIT = max(1, int(os.getenv("OPS_AGENT_QUEUE_FLUSH_LIMIT", "20") or 20))
|
||
_PERMANENT_DELIVERY_DETAIL_CODES = {
|
||
"agent_node_code_required",
|
||
"ops_job_not_owned_by_agent",
|
||
"ops_job_invalid_status",
|
||
}
|
||
_LAST_QUEUE_FLUSH_SUMMARY = {
|
||
"scanned": 0,
|
||
"delivered": 0,
|
||
"deferred": 0,
|
||
"dead_letter": 0,
|
||
"last_flush_at": "",
|
||
}
|
||
|
||
|
||
def _normalize_text_list(raw_value: object) -> list[str]:
|
||
"""Keep node-agent payload normalization aligned with release executor helpers."""
|
||
return _release_normalize_text_list(raw_value)
|
||
|
||
|
||
def _normalize_release_health_check_services(
|
||
payload: dict | None,
|
||
restart_services: list[str] | None,
|
||
) -> list[str]:
|
||
"""Backward-compatible wrapper used by node-agent tests and payload shaping."""
|
||
return _release_normalize_health_check_services(
|
||
dict(payload or {}),
|
||
list(restart_services or []),
|
||
default_api_service_name=API_SERVICE_NAME,
|
||
)
|
||
|
||
|
||
def _log(message: str) -> None:
|
||
print(f"{datetime.now().isoformat(sep=' ', timespec='seconds')} [node-agent] {message}", flush=True)
|
||
|
||
|
||
def _json_env(name: str, fallback: object) -> object:
|
||
raw = str(os.getenv(name, "")).strip()
|
||
if not raw:
|
||
return fallback
|
||
try:
|
||
return json.loads(raw)
|
||
except Exception:
|
||
return fallback
|
||
|
||
|
||
AGENT_CAPABILITIES = _json_env(
|
||
"OPS_AGENT_CAPABILITIES",
|
||
[
|
||
"service.start",
|
||
"service.stop",
|
||
"service.restart",
|
||
"service.status",
|
||
"runtime.start_worker",
|
||
"runtime.stop_worker",
|
||
"runtime.start_detection",
|
||
"runtime.stop_detection",
|
||
"runtime.pull_tasks",
|
||
"runtime.restart_api",
|
||
"runtime.start_sync_agent",
|
||
"runtime.stop_sync_agent",
|
||
"health.snapshot",
|
||
"logs.collect",
|
||
"diagnostics.collect",
|
||
"delivery.queue.flush",
|
||
"delivery.queue.replay",
|
||
"delivery.queue.discard",
|
||
"deploy.release",
|
||
],
|
||
)
|
||
AGENT_LABELS = _json_env("OPS_AGENT_LABELS", {})
|
||
|
||
|
||
class AgentResponseError(RuntimeError):
|
||
def __init__(self, message: str, *, detail_code: str = "", payload: dict | None = None):
|
||
super().__init__(message)
|
||
self.detail_code = str(detail_code or "").strip()
|
||
self.payload = dict(payload or {})
|
||
|
||
|
||
def _now_text() -> str:
|
||
return datetime.now().isoformat(sep=" ", timespec="seconds")
|
||
|
||
|
||
def _queue_pending_dir() -> str:
|
||
return os.path.join(AGENT_QUEUE_DIR, "pending")
|
||
|
||
|
||
def _queue_dead_letter_dir() -> str:
|
||
return os.path.join(AGENT_QUEUE_DIR, "dead-letter")
|
||
|
||
|
||
def _queue_discarded_dir() -> str:
|
||
return os.path.join(AGENT_QUEUE_DIR, "discarded")
|
||
|
||
|
||
def _ensure_queue_dirs() -> None:
|
||
for path in (AGENT_QUEUE_DIR, _queue_pending_dir(), _queue_dead_letter_dir(), _queue_discarded_dir()):
|
||
os.makedirs(path, exist_ok=True)
|
||
|
||
|
||
def _write_json_file(path: str, payload: dict) -> None:
|
||
temp_path = f"{path}.tmp"
|
||
with open(temp_path, "w", encoding="utf-8") as handle:
|
||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||
os.replace(temp_path, path)
|
||
|
||
|
||
def _read_json_file(path: str) -> dict:
|
||
with open(path, "r", encoding="utf-8") as handle:
|
||
return json.load(handle)
|
||
|
||
|
||
def _new_request_id(prefix: str, job_id: int) -> str:
|
||
normalized_prefix = str(prefix or "request").strip() or "request"
|
||
return f"{normalized_prefix}-{int(job_id or 0)}-{int(time.time() * 1000)}-{uuid4().hex[:10]}"
|
||
|
||
|
||
def _queue_file_names(directory: str) -> list[str]:
|
||
_ensure_queue_dirs()
|
||
try:
|
||
return sorted(file_name for file_name in os.listdir(directory) if file_name.endswith(".json"))
|
||
except FileNotFoundError:
|
||
return []
|
||
|
||
|
||
def _queue_head_record(directory: str) -> dict:
|
||
file_names = _queue_file_names(directory)
|
||
if not file_names:
|
||
return {}
|
||
file_path = os.path.join(directory, file_names[0])
|
||
try:
|
||
return _read_json_file(file_path)
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _queue_record_id(record: dict, *, file_name: str = "") -> str:
|
||
request_id = str((record or {}).get("request_id") or "").strip()
|
||
if request_id:
|
||
return request_id
|
||
normalized_file_name = str(file_name or "").strip()
|
||
if normalized_file_name.endswith(".json"):
|
||
return normalized_file_name[:-5]
|
||
return normalized_file_name
|
||
|
||
|
||
def _queue_entries(state: str) -> list[dict]:
|
||
normalized_state = str(state or "").strip()
|
||
if normalized_state == "pending":
|
||
directory = _queue_pending_dir()
|
||
elif normalized_state == "dead_letter":
|
||
directory = _queue_dead_letter_dir()
|
||
else:
|
||
return []
|
||
|
||
entries: list[dict] = []
|
||
for file_name in _queue_file_names(directory):
|
||
file_path = os.path.join(directory, file_name)
|
||
try:
|
||
record = _read_json_file(file_path)
|
||
except Exception:
|
||
continue
|
||
entries.append(
|
||
{
|
||
"state": normalized_state,
|
||
"file_name": file_name,
|
||
"file_path": file_path,
|
||
"record_id": _queue_record_id(record, file_name=file_name),
|
||
"record": dict(record or {}),
|
||
}
|
||
)
|
||
return entries
|
||
|
||
|
||
def _matches_delivery_selector(entry: dict, selector: dict, *, allowed_states: set[str] | None = None) -> bool:
|
||
normalized_selector = dict(selector or {})
|
||
record = dict(entry.get("record") or {})
|
||
state = str(entry.get("state") or "").strip()
|
||
if allowed_states and state not in allowed_states:
|
||
return False
|
||
|
||
selected_state = str(normalized_selector.get("state") or "").strip()
|
||
if selected_state and state != selected_state:
|
||
return False
|
||
|
||
selected_record_id = str(
|
||
normalized_selector.get("record_id")
|
||
or normalized_selector.get("request_id")
|
||
or ""
|
||
).strip()
|
||
if selected_record_id and str(entry.get("record_id") or "").strip() != selected_record_id:
|
||
return False
|
||
|
||
selected_request_kind = str(normalized_selector.get("request_kind") or "").strip()
|
||
if selected_request_kind and str(record.get("kind") or "").strip() != selected_request_kind:
|
||
return False
|
||
|
||
selected_detail_code = str(normalized_selector.get("detail_code") or "").strip()
|
||
if selected_detail_code and str(record.get("last_detail_code") or "").strip() != selected_detail_code:
|
||
return False
|
||
|
||
return True
|
||
|
||
|
||
def _trim_queue_preview(records: list[dict], *, limit: int = 5) -> list[dict]:
|
||
safe_limit = max(1, int(limit or 5))
|
||
preview: list[dict] = []
|
||
for entry in list(records or [])[:safe_limit]:
|
||
record = dict(entry.get("record") or {})
|
||
preview.append(
|
||
{
|
||
"record_id": str(entry.get("record_id") or "").strip(),
|
||
"state": str(entry.get("state") or "").strip(),
|
||
"request_kind": str(record.get("kind") or "").strip(),
|
||
"request_id": str(record.get("request_id") or "").strip(),
|
||
"detail_code": str(record.get("last_detail_code") or "").strip(),
|
||
"created_at": str(record.get("created_at") or "").strip(),
|
||
"updated_at": str(record.get("updated_at") or "").strip(),
|
||
}
|
||
)
|
||
return preview
|
||
|
||
|
||
def _normalize_queue_limit(raw_value: object, *, default: int = 20, minimum: int = 1, maximum: int = 200) -> int:
|
||
try:
|
||
value = int(raw_value or default)
|
||
except Exception:
|
||
value = int(default)
|
||
return max(minimum, min(maximum, value))
|
||
|
||
|
||
def _replay_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
selector = dict(normalized_payload.get("selector") or {})
|
||
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
|
||
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
|
||
flush_after_replay = bool(normalized_payload.get("flush_after_replay", True))
|
||
reason = str(normalized_payload.get("reason") or "").strip()
|
||
|
||
matched_entries = [
|
||
entry
|
||
for entry in _queue_entries("dead_letter")
|
||
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
|
||
][:limit]
|
||
if not matched_entries:
|
||
return False, "未找到符合条件的死信记录", {
|
||
"selector": selector,
|
||
"limit": limit,
|
||
"queue": _delivery_queue_snapshot(),
|
||
}
|
||
|
||
replayed_total = 0
|
||
for entry in matched_entries:
|
||
record = dict(entry.get("record") or {})
|
||
record["updated_at"] = _now_text()
|
||
record["replay_count"] = int(record.get("replay_count") or 0) + 1
|
||
record["last_replay_at"] = _now_text()
|
||
if reason:
|
||
record["last_replay_reason"] = reason
|
||
record.pop("dead_letter_at", None)
|
||
record.pop("dead_letter_reason", None)
|
||
_store_pending_delivery(record)
|
||
try:
|
||
os.remove(str(entry.get("file_path") or ""))
|
||
except FileNotFoundError:
|
||
pass
|
||
replayed_total += 1
|
||
|
||
flush_summary = {}
|
||
if flush_after_replay and replayed_total > 0:
|
||
flush_summary = _flush_delivery_queue(limit=replayed_total)
|
||
|
||
result = {
|
||
"selector": selector,
|
||
"limit": limit,
|
||
"replayed_total": replayed_total,
|
||
"flush_after_replay": flush_after_replay,
|
||
"flush_summary": flush_summary,
|
||
"matched_records": _trim_queue_preview(matched_entries),
|
||
"queue": _delivery_queue_snapshot(),
|
||
}
|
||
return True, f"已重放 {replayed_total} 条死信记录", result
|
||
|
||
|
||
def _store_discarded_delivery(record: dict, *, discarded_by: str = "", reason: str = "") -> str:
|
||
_ensure_queue_dirs()
|
||
discarded_record = dict(record or {})
|
||
discarded_record["discarded_at"] = _now_text()
|
||
discarded_record["discarded_by"] = str(discarded_by or "").strip()
|
||
discarded_record["discard_reason"] = str(reason or "").strip()
|
||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||
file_path = os.path.join(_queue_discarded_dir(), file_name)
|
||
_write_json_file(file_path, discarded_record)
|
||
return file_path
|
||
|
||
|
||
def _discard_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
selector = dict(normalized_payload.get("selector") or {})
|
||
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
|
||
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
|
||
discarded_by = str(normalized_payload.get("discarded_by") or "").strip()
|
||
reason = str(normalized_payload.get("reason") or "").strip()
|
||
|
||
matched_entries = [
|
||
entry
|
||
for entry in _queue_entries("dead_letter")
|
||
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
|
||
][:limit]
|
||
if not matched_entries:
|
||
return False, "未找到符合条件的死信记录", {
|
||
"selector": selector,
|
||
"limit": limit,
|
||
"queue": _delivery_queue_snapshot(),
|
||
}
|
||
|
||
discarded_total = 0
|
||
for entry in matched_entries:
|
||
_store_discarded_delivery(
|
||
dict(entry.get("record") or {}),
|
||
discarded_by=discarded_by,
|
||
reason=reason,
|
||
)
|
||
try:
|
||
os.remove(str(entry.get("file_path") or ""))
|
||
except FileNotFoundError:
|
||
pass
|
||
discarded_total += 1
|
||
|
||
result = {
|
||
"selector": selector,
|
||
"limit": limit,
|
||
"discarded_total": discarded_total,
|
||
"discarded_by": discarded_by,
|
||
"reason": reason,
|
||
"matched_records": _trim_queue_preview(matched_entries),
|
||
"queue": _delivery_queue_snapshot(),
|
||
}
|
||
return True, f"已丢弃 {discarded_total} 条死信记录", result
|
||
|
||
|
||
def _delivery_queue_snapshot() -> dict:
|
||
pending_dir = _queue_pending_dir()
|
||
dead_letter_dir = _queue_dead_letter_dir()
|
||
pending_files = _queue_file_names(pending_dir)
|
||
dead_letter_files = _queue_file_names(dead_letter_dir)
|
||
pending_head = _queue_head_record(pending_dir)
|
||
dead_letter_head = _queue_head_record(dead_letter_dir)
|
||
pending_count = len(pending_files)
|
||
dead_letter_count = len(dead_letter_files)
|
||
|
||
state = "healthy"
|
||
label = "正常"
|
||
reason = "当前没有待重试回执，也没有死信记录。"
|
||
if dead_letter_count > 0:
|
||
state = "dead_letter"
|
||
label = f"死信 {dead_letter_count}"
|
||
reason = "存在语义失败的回执/事件，自动重试已停止，建议人工查看。"
|
||
elif pending_count > 0:
|
||
state = "retrying"
|
||
label = f"待重试 {pending_count}"
|
||
reason = "存在待重试的回执/事件，Node Agent 会在后续 heartbeat/poll 周期继续回放。"
|
||
elif not pending_head and not dead_letter_head:
|
||
label = "正常"
|
||
|
||
return {
|
||
"state": state,
|
||
"label": label,
|
||
"reason": reason,
|
||
"pending_count": pending_count,
|
||
"dead_letter_count": dead_letter_count,
|
||
"oldest_pending_at": str(pending_head.get("created_at") or "").strip(),
|
||
"oldest_pending_request_id": str(pending_head.get("request_id") or "").strip(),
|
||
"oldest_pending_kind": str(pending_head.get("kind") or "").strip(),
|
||
"oldest_dead_letter_at": str(
|
||
dead_letter_head.get("dead_letter_at") or dead_letter_head.get("created_at") or ""
|
||
).strip(),
|
||
"oldest_dead_letter_request_id": str(dead_letter_head.get("request_id") or "").strip(),
|
||
"oldest_dead_letter_kind": str(dead_letter_head.get("kind") or "").strip(),
|
||
"last_flush_at": str(_LAST_QUEUE_FLUSH_SUMMARY.get("last_flush_at") or "").strip(),
|
||
"last_flush_delivered": int(_LAST_QUEUE_FLUSH_SUMMARY.get("delivered") or 0),
|
||
"last_flush_deferred": int(_LAST_QUEUE_FLUSH_SUMMARY.get("deferred") or 0),
|
||
"last_flush_dead_letter": int(_LAST_QUEUE_FLUSH_SUMMARY.get("dead_letter") or 0),
|
||
}
|
||
|
||
|
||
def _response_detail_code(response: dict) -> str:
|
||
if not isinstance(response, dict):
|
||
return ""
|
||
top_level = str(response.get("detail_code") or "").strip()
|
||
if top_level:
|
||
return top_level
|
||
data = response.get("data")
|
||
if isinstance(data, dict):
|
||
return str(data.get("detail_code") or "").strip()
|
||
return ""
|
||
|
||
|
||
def _ensure_ok_response(response: dict, fallback_message: str) -> dict:
|
||
raw_code = response.get("code", 1)
|
||
try:
|
||
normalized_code = int(raw_code if raw_code not in (None, "") else 1)
|
||
except Exception:
|
||
normalized_code = 1
|
||
if normalized_code == 0:
|
||
return response
|
||
raise AgentResponseError(
|
||
str(response.get("message") or fallback_message),
|
||
detail_code=_response_detail_code(response),
|
||
payload=(response.get("data") if isinstance(response.get("data"), dict) else {}),
|
||
)
|
||
|
||
|
||
def _build_delivery_record(kind: str, path: str, payload: dict, request_id: str) -> dict:
|
||
return {
|
||
"kind": str(kind or "").strip() or "delivery",
|
||
"path": str(path or "").strip(),
|
||
"payload": dict(payload or {}),
|
||
"request_id": str(request_id or "").strip(),
|
||
"attempt_count": 0,
|
||
"created_at": _now_text(),
|
||
"updated_at": _now_text(),
|
||
"last_error": "",
|
||
"last_detail_code": "",
|
||
}
|
||
|
||
|
||
def _store_pending_delivery(record: dict) -> str:
|
||
_ensure_queue_dirs()
|
||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||
file_path = os.path.join(_queue_pending_dir(), file_name)
|
||
_write_json_file(file_path, record)
|
||
return file_path
|
||
|
||
|
||
def _store_dead_letter_delivery(record: dict, *, reason: str, detail_code: str = "") -> str:
|
||
_ensure_queue_dirs()
|
||
dead_record = dict(record or {})
|
||
dead_record["dead_letter_reason"] = str(reason or "").strip()
|
||
dead_record["dead_letter_at"] = _now_text()
|
||
dead_record["last_error"] = str(reason or "").strip()
|
||
dead_record["last_detail_code"] = str(detail_code or "").strip()
|
||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||
file_path = os.path.join(_queue_dead_letter_dir(), file_name)
|
||
_write_json_file(file_path, dead_record)
|
||
return file_path
|
||
|
||
|
||
def _dispatch_delivery_record(record: dict) -> dict:
|
||
kind = str(record.get("kind") or "delivery").strip() or "delivery"
|
||
path = str(record.get("path") or "").strip()
|
||
payload = record.get("payload") or {}
|
||
timeout = 15 if kind == "job_event" else 30
|
||
response = _post(path, payload, timeout=timeout)
|
||
return _ensure_ok_response(response, f"{kind} failed")
|
||
|
||
|
||
def _flush_delivery_queue(limit: int | None = None) -> dict:
|
||
global _LAST_QUEUE_FLUSH_SUMMARY
|
||
_ensure_queue_dirs()
|
||
safe_limit = max(1, int(limit or AGENT_QUEUE_FLUSH_LIMIT or 1))
|
||
summary = {
|
||
"scanned": 0,
|
||
"delivered": 0,
|
||
"deferred": 0,
|
||
"dead_letter": 0,
|
||
}
|
||
file_names = sorted(
|
||
file_name
|
||
for file_name in os.listdir(_queue_pending_dir())
|
||
if file_name.endswith(".json")
|
||
)[:safe_limit]
|
||
for file_name in file_names:
|
||
summary["scanned"] += 1
|
||
file_path = os.path.join(_queue_pending_dir(), file_name)
|
||
try:
|
||
record = _read_json_file(file_path)
|
||
except Exception as exc:
|
||
_store_dead_letter_delivery({"file_name": file_name}, reason=f"invalid queue record: {exc}")
|
||
try:
|
||
os.remove(file_path)
|
||
except FileNotFoundError:
|
||
pass
|
||
summary["dead_letter"] += 1
|
||
continue
|
||
|
||
try:
|
||
_dispatch_delivery_record(record)
|
||
try:
|
||
os.remove(file_path)
|
||
except FileNotFoundError:
|
||
pass
|
||
summary["delivered"] += 1
|
||
except AgentResponseError as exc:
|
||
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
|
||
record["updated_at"] = _now_text()
|
||
record["last_error"] = str(exc)
|
||
record["last_detail_code"] = exc.detail_code
|
||
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
|
||
_store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
|
||
try:
|
||
os.remove(file_path)
|
||
except FileNotFoundError:
|
||
pass
|
||
summary["dead_letter"] += 1
|
||
_log(
|
||
f"delivery moved to dead-letter: kind={record.get('kind')} request_id={record.get('request_id')} "
|
||
f"detail_code={exc.detail_code or '-'} message={exc}"
|
||
)
|
||
else:
|
||
_write_json_file(file_path, record)
|
||
summary["deferred"] += 1
|
||
except Exception as exc:
|
||
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
|
||
record["updated_at"] = _now_text()
|
||
record["last_error"] = str(exc)
|
||
_write_json_file(file_path, record)
|
||
summary["deferred"] += 1
|
||
summary["last_flush_at"] = _now_text()
|
||
_LAST_QUEUE_FLUSH_SUMMARY = dict(summary)
|
||
return summary
|
||
|
||
|
||
def _deliver_or_queue(
|
||
*,
|
||
kind: str,
|
||
path: str,
|
||
payload: dict,
|
||
request_id: str,
|
||
timeout: int = 30,
|
||
) -> dict:
|
||
record = _build_delivery_record(kind, path, payload, request_id)
|
||
try:
|
||
response = _post(path, payload, timeout=timeout)
|
||
_ensure_ok_response(response, f"{kind} failed")
|
||
return {"state": "delivered", "request_id": request_id, "detail_code": "", "response": response}
|
||
except AgentResponseError as exc:
|
||
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
|
||
file_path = _store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
|
||
_log(
|
||
f"{kind} dead-lettered: request_id={request_id} detail_code={exc.detail_code or '-'} "
|
||
f"message={exc} file={file_path}"
|
||
)
|
||
return {"state": "dead_letter", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
|
||
file_path = _store_pending_delivery({**record, "last_error": str(exc), "last_detail_code": exc.detail_code})
|
||
_log(
|
||
f"{kind} queued for retry: request_id={request_id} detail_code={exc.detail_code or '-'} "
|
||
f"message={exc} file={file_path}"
|
||
)
|
||
return {"state": "queued", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
|
||
except Exception as exc:
|
||
file_path = _store_pending_delivery({**record, "last_error": str(exc)})
|
||
_log(f"{kind} queued for retry: request_id={request_id} message={exc} file={file_path}")
|
||
return {"state": "queued", "request_id": request_id, "detail_code": "", "file_path": file_path}
|
||
|
||
|
||
def _headers() -> dict[str, str]:
|
||
return {
|
||
"Content-Type": "application/json",
|
||
"X-Domaincheck-Agent-Token": AGENT_TOKEN,
|
||
}
|
||
|
||
|
||
def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict:
|
||
if not CONTROL_PLANE_BASE_URL:
|
||
raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置")
|
||
if not AGENT_TOKEN:
|
||
raise RuntimeError("OPS_AGENT_TOKEN 未配置")
|
||
url = f"{CONTROL_PLANE_BASE_URL}{path}"
|
||
data = json.dumps(payload or {}, ensure_ascii=False).encode("utf-8")
|
||
request = urllib.request.Request(url=url, data=data, headers=_headers(), method=method.upper())
|
||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||
body = response.read().decode("utf-8", errors="ignore")
|
||
return json.loads(body or "{}")
|
||
|
||
|
||
def _post(path: str, payload: dict, timeout: int = 30) -> dict:
|
||
return _request("POST", path, payload, timeout=timeout)
|
||
|
||
|
||
def _hostname() -> str:
|
||
try:
|
||
return socket.gethostname()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def _ip() -> str:
|
||
try:
|
||
return socket.gethostbyname(socket.gethostname())
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def _base_payload() -> dict:
|
||
return {
|
||
"node_code": NODE_CODE,
|
||
"region": NODE_REGION,
|
||
"role": NODE_ROLE,
|
||
"title": NODE_CODE,
|
||
"hostname": _hostname(),
|
||
"ip": _ip(),
|
||
"agent_version": AGENT_VERSION,
|
||
"capabilities": AGENT_CAPABILITIES,
|
||
"labels": AGENT_LABELS,
|
||
"metadata": {
|
||
"service_names": {
|
||
"api": API_SERVICE_NAME,
|
||
"worker": WORKER_SERVICE_NAME,
|
||
"sync_agent": SYNC_AGENT_SERVICE_NAME,
|
||
"node_agent": NODE_AGENT_SERVICE_NAME,
|
||
},
|
||
"delivery_queue": _delivery_queue_snapshot(),
|
||
},
|
||
}
|
||
|
||
|
||
def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]:
|
||
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||
return completed.returncode, completed.stdout.strip(), completed.stderr.strip()
|
||
|
||
|
||
def _execute_action(
|
||
action: str,
|
||
payload: dict,
|
||
*,
|
||
job_id: int | None = None,
|
||
job_context: dict | None = None,
|
||
) -> tuple[bool, str, dict]:
|
||
normalized_action = str(action or "").strip()
|
||
if normalized_action == "delivery.queue.flush":
|
||
limit = _normalize_queue_limit((payload or {}).get("limit"), default=20)
|
||
summary = _flush_delivery_queue(limit=limit)
|
||
return True, "Delivery Queue 已执行冲刷", {
|
||
"action": normalized_action,
|
||
"limit": limit,
|
||
"flush_summary": summary,
|
||
"queue": _delivery_queue_snapshot(),
|
||
}
|
||
if normalized_action == "delivery.queue.replay":
|
||
return _replay_dead_letter_records(payload)
|
||
if normalized_action == "delivery.queue.discard":
|
||
return _discard_dead_letter_records(payload)
|
||
if supports_structured_action(normalized_action):
|
||
return execute_structured_action(
|
||
normalized_action,
|
||
payload,
|
||
service_names=build_service_name_map(
|
||
api_service_name=API_SERVICE_NAME,
|
||
worker_service_name=WORKER_SERVICE_NAME,
|
||
sync_agent_service_name=SYNC_AGENT_SERVICE_NAME,
|
||
node_agent_service_name=NODE_AGENT_SERVICE_NAME,
|
||
),
|
||
runner=_run,
|
||
host_context={
|
||
"hostname": _hostname(),
|
||
"ip": _ip(),
|
||
},
|
||
)
|
||
if normalized_action == "deploy.release":
|
||
return execute_release_action(
|
||
dict(payload or {}),
|
||
run_command=_run,
|
||
default_api_service_name=API_SERVICE_NAME,
|
||
event_callback=(
|
||
(lambda event_type, message, level="info", payload=None: _job_event(
|
||
int(job_id),
|
||
event_type=event_type,
|
||
message=message,
|
||
level=level,
|
||
payload={
|
||
**dict(payload or {}),
|
||
"release_context": dict((job_context or {}).get("release_context") or {}),
|
||
"step_key": str((job_context or {}).get("step_key") or "").strip(),
|
||
},
|
||
summary_text=message,
|
||
focus_ref=dict((job_context or {}).get("focus_ref") or {}),
|
||
occurred_at=_now_text(),
|
||
))
|
||
if job_id
|
||
else None
|
||
),
|
||
urlopen_func=urllib.request.urlopen,
|
||
user_agent=f"domaincheck-node-agent/{AGENT_VERSION}",
|
||
)
|
||
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}
|
||
|
||
|
||
def _register() -> None:
|
||
response = _post("/api/v1/ops/agent/register", _base_payload())
|
||
_ensure_ok_response(response, "agent register failed")
|
||
_log(f"registered: {response.get('message')}")
|
||
|
||
|
||
def _heartbeat() -> None:
|
||
response = _post("/api/v1/ops/agent/heartbeat", _base_payload())
|
||
_ensure_ok_response(response, "agent heartbeat failed")
|
||
|
||
|
||
def _pull_jobs() -> list[dict]:
|
||
response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE})
|
||
_ensure_ok_response(response, "agent pull failed")
|
||
data = response.get("data") or {}
|
||
return list(data.get("jobs") or [])
|
||
|
||
|
||
def _normalize_agent_job(job: dict) -> dict:
|
||
normalized_job = dict(job or {})
|
||
focus_ref = normalized_job.get("focus_ref") if isinstance(normalized_job.get("focus_ref"), dict) else {}
|
||
release_context = (
|
||
normalized_job.get("release_context") if isinstance(normalized_job.get("release_context"), dict) else {}
|
||
)
|
||
step_ref = normalized_job.get("step_ref") if isinstance(normalized_job.get("step_ref"), dict) else {}
|
||
job_id = int(normalized_job.get("job_id") or normalized_job.get("id") or 0)
|
||
step_key = str(
|
||
normalized_job.get("step_key")
|
||
or step_ref.get("step_key")
|
||
or "dispatch"
|
||
).strip() or "dispatch"
|
||
step_title = str(
|
||
normalized_job.get("step_title")
|
||
or step_ref.get("step_title")
|
||
or normalized_job.get("summary")
|
||
or normalized_job.get("action")
|
||
or step_key
|
||
).strip() or step_key
|
||
if not focus_ref:
|
||
focus_ref = {
|
||
"kind": "ops_job",
|
||
"job_id": job_id,
|
||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||
"action": str(normalized_job.get("action") or "").strip(),
|
||
"target_node_code": str(normalized_job.get("target_node_code") or "").strip(),
|
||
}
|
||
return {
|
||
**normalized_job,
|
||
"job_id": job_id,
|
||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||
"job_type": str(normalized_job.get("job_type") or "ops_action").strip() or "ops_action",
|
||
"action": str(normalized_job.get("action") or "").strip(),
|
||
"payload": dict(normalized_job.get("payload") or {}),
|
||
"policy": dict(normalized_job.get("policy") or {}),
|
||
"focus_ref": focus_ref,
|
||
"release_context": dict(release_context or {}),
|
||
"step_key": step_key,
|
||
"step_title": step_title,
|
||
"step_ref": {
|
||
"step_id": int(step_ref.get("step_id") or 0),
|
||
"step_key": step_key,
|
||
"step_title": step_title,
|
||
},
|
||
}
|
||
|
||
|
||
def _job_start(job_id: int, *, job: dict | None = None) -> None:
|
||
normalized_job = _normalize_agent_job(job or {"job_id": job_id})
|
||
response = _post(
|
||
f"/api/v1/ops/agent/jobs/{job_id}/start",
|
||
{
|
||
"node_code": NODE_CODE,
|
||
"job_code": normalized_job.get("job_code") or "",
|
||
"action": normalized_job.get("action") or "",
|
||
"step_key": normalized_job.get("step_key") or "",
|
||
"focus_ref": normalized_job.get("focus_ref") or {},
|
||
},
|
||
)
|
||
_ensure_ok_response(response, "job start failed")
|
||
|
||
|
||
def _job_complete(
|
||
job_id: int,
|
||
*,
|
||
status: str,
|
||
stdout: str,
|
||
stderr: str,
|
||
result: dict,
|
||
error_message: str = "",
|
||
client_request_id: str | None = None,
|
||
duration_ms: int | None = None,
|
||
summary_text: str = "",
|
||
focus_ref: dict | None = None,
|
||
step_ref: dict | None = None,
|
||
release_context: dict | None = None,
|
||
) -> dict:
|
||
request_id = str(client_request_id or "").strip() or _new_request_id("complete", job_id)
|
||
payload = {
|
||
"node_code": NODE_CODE,
|
||
"status": status,
|
||
"stdout": stdout,
|
||
"stderr": stderr,
|
||
"result": result,
|
||
"error_message": error_message,
|
||
"client_request_id": request_id,
|
||
}
|
||
if duration_ms is not None:
|
||
payload["duration_ms"] = max(0, int(duration_ms or 0))
|
||
if str(summary_text or "").strip():
|
||
payload["summary_text"] = str(summary_text).strip()
|
||
if isinstance(focus_ref, dict) and focus_ref:
|
||
payload["focus_ref"] = dict(focus_ref)
|
||
if isinstance(step_ref, dict) and step_ref:
|
||
payload["step_ref"] = dict(step_ref)
|
||
if isinstance(release_context, dict) and release_context:
|
||
payload["release_context"] = dict(release_context)
|
||
return _deliver_or_queue(
|
||
kind="job_complete",
|
||
path=f"/api/v1/ops/agent/jobs/{job_id}/complete",
|
||
payload=payload,
|
||
request_id=request_id,
|
||
timeout=30,
|
||
)
|
||
|
||
|
||
def _job_event(
|
||
job_id: int,
|
||
*,
|
||
event_type: str,
|
||
message: str,
|
||
level: str = "info",
|
||
payload: dict | None = None,
|
||
client_event_id: str | None = None,
|
||
summary_text: str = "",
|
||
focus_ref: dict | None = None,
|
||
occurred_at: str = "",
|
||
) -> dict:
|
||
request_id = str(client_event_id or "").strip() or _new_request_id("event", job_id)
|
||
delivery_payload = {
|
||
"node_code": NODE_CODE,
|
||
"event_type": event_type,
|
||
"message": message,
|
||
"level": level,
|
||
"payload": payload or {},
|
||
"client_event_id": request_id,
|
||
}
|
||
if str(summary_text or "").strip():
|
||
delivery_payload["summary_text"] = str(summary_text).strip()
|
||
if isinstance(focus_ref, dict) and focus_ref:
|
||
delivery_payload["focus_ref"] = dict(focus_ref)
|
||
if str(occurred_at or "").strip():
|
||
delivery_payload["occurred_at"] = str(occurred_at).strip()
|
||
return _deliver_or_queue(
|
||
kind="job_event",
|
||
path=f"/api/v1/ops/agent/jobs/{job_id}/events",
|
||
payload=delivery_payload,
|
||
request_id=request_id,
|
||
timeout=15,
|
||
)
|
||
|
||
|
||
def _process_job(job: dict) -> None:
|
||
normalized_job = _normalize_agent_job(job)
|
||
job_id = int(normalized_job.get("job_id") or 0)
|
||
action = str(normalized_job.get("action") or "").strip()
|
||
payload = dict(normalized_job.get("payload") or {})
|
||
if job_id <= 0 or not action:
|
||
return
|
||
|
||
started_at = time.monotonic()
|
||
_log(
|
||
"job start: "
|
||
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
|
||
f"type={normalized_job.get('job_type') or '-'} "
|
||
f"step={normalized_job.get('step_key') or '-'} action={action}"
|
||
)
|
||
start_delivery_state = "delivered"
|
||
start_delivery_error = ""
|
||
try:
|
||
_job_start(job_id, job=normalized_job)
|
||
except Exception as exc:
|
||
start_delivery_state = "failed_local"
|
||
start_delivery_error = str(exc)
|
||
_log(
|
||
"job start delivery failed, continue locally: "
|
||
f"id={job_id} code={normalized_job.get('job_code') or '-'} action={action} error={exc}"
|
||
)
|
||
_job_event(
|
||
job_id,
|
||
event_type="executor_received",
|
||
message=f"node agent accepted action {action}",
|
||
summary_text=f"已接单 {action}",
|
||
focus_ref=dict(normalized_job.get("focus_ref") or {}),
|
||
occurred_at=_now_text(),
|
||
payload={
|
||
"action": action,
|
||
"job_code": normalized_job.get("job_code") or "",
|
||
"job_type": normalized_job.get("job_type") or "",
|
||
"step_key": normalized_job.get("step_key") or "",
|
||
"step_title": normalized_job.get("step_title") or "",
|
||
"release_context": dict(normalized_job.get("release_context") or {}),
|
||
"start_delivery_state": start_delivery_state,
|
||
"start_delivery_error": start_delivery_error,
|
||
},
|
||
)
|
||
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
|
||
stdout = str(result.get("stdout") or "")
|
||
stderr = str(result.get("stderr") or "")
|
||
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
|
||
summary_text = str(result.get("summary_text") or result.get("summary") or message).strip()
|
||
delivery = _job_complete(
|
||
job_id,
|
||
status="success" if ok else "failed",
|
||
stdout=stdout,
|
||
stderr=stderr,
|
||
result=result,
|
||
error_message="" if ok else message,
|
||
duration_ms=duration_ms,
|
||
summary_text=summary_text,
|
||
focus_ref=dict(normalized_job.get("focus_ref") or {}),
|
||
step_ref=dict(normalized_job.get("step_ref") or {}),
|
||
release_context=dict(normalized_job.get("release_context") or {}),
|
||
)
|
||
_log(
|
||
"job complete: "
|
||
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
|
||
f"action={action} ok={ok} duration_ms={duration_ms} delivery={delivery.get('state')}"
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
if not NODE_CODE:
|
||
raise RuntimeError("NODE_CODE 未配置")
|
||
_log(f"starting node agent: node={NODE_CODE} role={NODE_ROLE} region={NODE_REGION}")
|
||
_ensure_queue_dirs()
|
||
_register()
|
||
last_heartbeat_at = 0.0
|
||
|
||
while True:
|
||
now = time.time()
|
||
try:
|
||
delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT)
|
||
if delivery_summary["delivered"] or delivery_summary["dead_letter"]:
|
||
_log(f"delivery queue flush: {delivery_summary}")
|
||
if now - last_heartbeat_at >= 15:
|
||
_heartbeat()
|
||
last_heartbeat_at = now
|
||
jobs = _pull_jobs()
|
||
if jobs:
|
||
for job in jobs:
|
||
_process_job(job)
|
||
else:
|
||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||
except urllib.error.HTTPError as exc:
|
||
_log(f"http error: {exc.code}")
|
||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||
except urllib.error.URLError as exc:
|
||
_log(f"url error: {exc}")
|
||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||
except Exception as exc:
|
||
_log(f"loop error: {exc}")
|
||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|