admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

435
domain-api/app/node_agent.py
View File

@@ -1,13 +1,16 @@
from __future__ import annotations
import hashlib
import json
import os
import socket
import subprocess
import traceback
import time
import urllib.error
import urllib.request
from datetime import datetime
from urllib.parse import urlparse
from uuid import uuid4
from app.services.ops_action_executor_core import (
@@ -28,6 +31,44 @@ NODE_CODE = str(os.getenv("NODE_CODE", "")).strip()
NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland"
NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker"
AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5))
AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS", "30") or 30),
)
AGENT_HTTP_TIMEOUT_SECONDS = max(10, int(os.getenv("OPS_AGENT_HTTP_TIMEOUT_SECONDS", "60") or 60))
AGENT_REGISTER_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_REGISTER_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_HEARTBEAT_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_HEARTBEAT_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_PULL_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_PULL_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS))
or AGENT_HTTP_TIMEOUT_SECONDS
),
)
AGENT_JOB_COMPLETE_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_JOB_COMPLETE_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS))
or AGENT_HTTP_TIMEOUT_SECONDS
),
)
AGENT_JOB_EVENT_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_JOB_EVENT_TIMEOUT_SECONDS", str(max(15, AGENT_HTTP_TIMEOUT_SECONDS // 2)))
or max(15, AGENT_HTTP_TIMEOUT_SECONDS // 2)
),
)
WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker"
API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api"
SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent"
@@ -50,6 +91,7 @@ _LAST_QUEUE_FLUSH_SUMMARY = {
"dead_letter": 0,
"last_flush_at": "",
}
_LAST_RUNTIME_CONFIG_HASH = ""
def _normalize_text_list(raw_value: object) -> list[str]:
@@ -98,6 +140,7 @@ AGENT_CAPABILITIES = _json_env(
"runtime.restart_api",
"runtime.start_sync_agent",
"runtime.stop_sync_agent",
"runtime.reset_lab_state",
"health.snapshot",
"logs.collect",
"diagnostics.collect",
@@ -601,7 +644,7 @@ def _headers() -> dict[str, str]:
}
def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict:
def _request(method: str, path: str, payload: dict | None = None, timeout: int = AGENT_HTTP_TIMEOUT_SECONDS) -> dict:
if not CONTROL_PLANE_BASE_URL:
raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置")
if not AGENT_TOKEN:
@@ -614,22 +657,286 @@ def _request(method: str, path: str, payload: dict | None = None, timeout: int =
return json.loads(body or "{}")
def _post(path: str, payload: dict, timeout: int = 30) -> dict:
def _post(path: str, payload: dict, timeout: int = AGENT_HTTP_TIMEOUT_SECONDS) -> dict:
return _request("POST", path, payload, timeout=timeout)
def _hostname() -> str:
def _pull_runtime_config(timeout: int = AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS) -> dict:
response = _post("/api/v1/ops/agent/runtime-config", {"node_code": NODE_CODE}, timeout=timeout)
_ensure_ok_response(response, "agent runtime config pull failed")
data = response.get("data") or {}
return dict(data.get("bundle") or {})
def _publish_local_config_update(config_type: str) -> None:
try:
return socket.gethostname()
from app.core.redis_client import get_redis
redis_client = get_redis()
redis_client.publish("domain_tool:config_update", str(config_type or "").strip() or "config")
except Exception:
return ""
return
def _apply_runtime_config(bundle: dict) -> bool:
global _LAST_RUNTIME_CONFIG_HASH
normalized_bundle = dict(bundle or {})
bundle_hash = str(normalized_bundle.get("config_hash") or "").strip()
if not bundle_hash:
bundle_hash = hashlib.sha256(
json.dumps(normalized_bundle, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
if bundle_hash and bundle_hash == _LAST_RUNTIME_CONFIG_HASH:
return False
from app.core.files import write_json
from app.services.runtime_settings_service import update_runtime_settings
from app.services.sensitive_words_service import save_sensitive_words_payload
detect_options = dict(normalized_bundle.get("detect_options") or {})
proxy_config = dict(normalized_bundle.get("proxy_config") or {})
thread_count = int(normalized_bundle.get("thread_count", 2) or 2)
node_thread_counts = dict(normalized_bundle.get("node_thread_counts") or {})
runtime_settings = dict(normalized_bundle.get("runtime_settings") or {})
sensitive_words = dict(normalized_bundle.get("sensitive_words") or {})
sensitive_words_text = str(sensitive_words.get("text") or "")
sensitive_word_items = list(sensitive_words.get("items") or [])
write_json("detect_options.json", detect_options)
write_json("proxy_config.json", proxy_config)
write_json("thread_count.json", {"thread_count": str(thread_count)})
write_json("node_thread_counts.json", node_thread_counts)
write_json("runtime_settings.json", runtime_settings)
write_json("runtime/runtime_settings.json", runtime_settings)
write_json(
"runtime/sensitive_words.json",
{
"items": sensitive_word_items,
"text": sensitive_words_text,
"total": int(sensitive_words.get("total", 0) or 0),
},
)
try:
update_runtime_settings(runtime_settings)
except Exception as exc:
_log(f"runtime settings local api sync skipped: {exc}")
try:
save_sensitive_words_payload({"text": sensitive_words_text})
except Exception as exc:
_log(f"sensitive words db sync skipped: {exc}")
try:
from app.core.redis_client import get_redis
redis_client = get_redis()
redis_client.set("domain_tool:detect_options", json.dumps(detect_options, ensure_ascii=False))
redis_client.set("domain_tool:proxy_config", json.dumps(proxy_config, ensure_ascii=False))
redis_client.set("domain_tool:thread_count", thread_count)
redis_client.set("domain_tool:node_thread_counts", json.dumps(node_thread_counts, ensure_ascii=False))
redis_client.set("domain_tool:runtime_settings", json.dumps(runtime_settings, ensure_ascii=False))
redis_client.set("domain_tool:sensitive_words", json.dumps(sensitive_word_items, ensure_ascii=False))
redis_client.publish("domain_tool:config_update", "detect_options")
redis_client.publish("domain_tool:config_update", "proxy_config")
redis_client.publish("domain_tool:config_update", "thread_count")
redis_client.publish("domain_tool:config_update", "node_thread_counts")
redis_client.publish("domain_tool:config_update", "runtime_settings")
redis_client.publish("domain_tool:config_update", "sensitive_words")
except Exception:
_publish_local_config_update("detect_options")
_publish_local_config_update("proxy_config")
_publish_local_config_update("thread_count")
_publish_local_config_update("node_thread_counts")
_publish_local_config_update("runtime_settings")
_publish_local_config_update("sensitive_words")
_LAST_RUNTIME_CONFIG_HASH = bundle_hash
_log(
"runtime config applied: "
f"thread_count={thread_count} "
f"node_override={node_thread_counts.get(NODE_CODE)} "
f"sensitive_words={int(sensitive_words.get('total', 0) or 0)}"
)
return True
def _hostname() -> str:
generic_values = {"localhost", "localhost.localdomain", "ip6-localhost", "localhost6"}
candidates: list[str] = []
try:
candidates.append(socket.gethostname())
except Exception:
pass
try:
candidates.append(socket.getfqdn())
except Exception:
pass
try:
candidates.append(os.uname().nodename)
except Exception:
pass
candidates.append(str(os.getenv("HOSTNAME", "")).strip())
for value in candidates:
normalized = str(value or "").strip()
if not normalized:
continue
if normalized.lower() in generic_values:
continue
return normalized
return NODE_CODE or ""
def _control_plane_host_port() -> tuple[str, int]:
parsed = urlparse(CONTROL_PLANE_BASE_URL if "://" in CONTROL_PLANE_BASE_URL else f"http://{CONTROL_PLANE_BASE_URL}")
host = str(parsed.hostname or "").strip()
if not host:
return "", 0
port = int(parsed.port or (443 if parsed.scheme == "https" else 80))
return host, port
def _first_non_loopback_ip(values: list[str]) -> str:
for value in values:
normalized = str(value or "").strip()
if not normalized or normalized.startswith("127.") or normalized == "::1":
continue
return normalized
return ""
def _ip() -> str:
explicit_ip = str(os.getenv("OPS_AGENT_IP", "")).strip()
if explicit_ip:
return explicit_ip
host, port = _control_plane_host_port()
if host and port:
for family in (socket.AF_INET, socket.AF_INET6):
sock = None
try:
sock = socket.socket(family, socket.SOCK_DGRAM)
sock.connect((host, port))
local_ip = str(sock.getsockname()[0] or "").strip()
if local_ip and not local_ip.startswith("127.") and local_ip != "::1":
return local_ip
except Exception:
pass
finally:
if sock is not None:
try:
sock.close()
except Exception:
pass
hostname = _hostname()
if hostname:
try:
addrinfo = socket.getaddrinfo(hostname, None)
resolved_ips = [str(item[4][0] or "").strip() for item in addrinfo if item and len(item) >= 5]
best_ip = _first_non_loopback_ip(resolved_ips)
if best_ip:
return best_ip
except Exception:
pass
try:
return socket.gethostbyname(socket.gethostname())
fallback_ip = str(socket.gethostbyname(socket.gethostname()) or "").strip()
if fallback_ip and not fallback_ip.startswith("127."):
return fallback_ip
except Exception:
return ""
pass
return ""
def _detect_runtime_snapshot() -> dict:
worker_runtime = {}
try:
from app.services.worker_control_service import detect_worker_runtime
worker_runtime = detect_worker_runtime() or {}
except Exception as exc:
worker_runtime = {
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": "",
"error": str(exc),
}
try:
from app.services.detect_service import get_detect_status
detect_status = get_detect_status() or {}
runtime_state = detect_status.get("runtime_state") if isinstance(detect_status.get("runtime_state"), dict) else {}
active_job = detect_status.get("active_job") if isinstance(detect_status.get("active_job"), dict) else {}
active_threads = max(0, int(detect_status.get("active_thread_count", 0) or 0))
max_threads = max(0, int(detect_status.get("max_thread_count", 0) or 0))
current_load = max(
active_threads,
int(active_job.get("items_running", 0) or 0),
)
phase_label = str(detect_status.get("phase_label") or "").strip()
phase_detail = str(
detect_status.get("phase_detail")
or detect_status.get("recent_event")
or detect_status.get("worker_runtime_message")
or ""
).strip()
inferred_worker_online = bool(
worker_runtime.get("running", False)
or detect_status.get("worker_online", False)
or runtime_state.get("service_running", False)
or current_load > 0
or active_threads > 0
)
return {
"worker_online": inferred_worker_online,
"service_running": bool(
runtime_state.get("service_running", False)
or worker_runtime.get("running", False)
or inferred_worker_online
),
"detecting": bool(
detect_status.get("detecting", False)
or runtime_state.get("detecting", False)
or current_load > 0
or active_threads > 0
),
"active_threads": active_threads,
"max_threads": max_threads,
"current_load": current_load,
"phase_label": phase_label,
"phase_detail": phase_detail,
"recent_warning": str(detect_status.get("recent_warning") or "").strip(),
"updated_at": str(runtime_state.get("updated_at") or "").strip(),
"detect_participating": bool(
detect_status.get("detect_participating", False)
or current_load > 0
or active_threads > 0
),
}
except Exception as exc:
worker_online = bool(worker_runtime.get("running", False))
worker_message = str(worker_runtime.get("message") or "").strip()
return {
"worker_online": worker_online,
"service_running": worker_online,
"detecting": False,
"active_threads": 0,
"max_threads": 0,
"current_load": 0,
"phase_label": "",
"phase_detail": worker_message,
"recent_warning": "",
"updated_at": str(worker_runtime.get("latest_start_time") or "").strip(),
"detect_participating": False,
"error": str(exc),
}
def _base_payload() -> dict:
@@ -651,12 +958,37 @@ def _base_payload() -> dict:
"node_agent": NODE_AGENT_SERVICE_NAME,
},
"delivery_queue": _delivery_queue_snapshot(),
"detect_runtime": _detect_runtime_snapshot(),
},
}
def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]:
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
normalized_command = [str(part or "").strip() for part in command]
combined_output = f"{completed.stdout or ''}\n{completed.stderr or ''}".lower()
needs_sudo_retry = (
normalized_command
and normalized_command[0] == "systemctl"
and completed.returncode != 0
and "sudo" not in normalized_command
and any(
marker in combined_output
for marker in (
"interactive authentication required",
"authentication is required",
"authorization not available",
"polkit",
)
)
)
if needs_sudo_retry:
completed = subprocess.run(
["sudo", "-n", *normalized_command],
capture_output=True,
text=True,
timeout=timeout,
)
return completed.returncode, completed.stdout.strip(), completed.stderr.strip()
@@ -727,18 +1059,22 @@ def _execute_action(
def _register() -> None:
response = _post("/api/v1/ops/agent/register", _base_payload())
response = _post("/api/v1/ops/agent/register", _base_payload(), timeout=AGENT_REGISTER_TIMEOUT_SECONDS)
_ensure_ok_response(response, "agent register failed")
_log(f"registered: {response.get('message')}")
def _heartbeat() -> None:
response = _post("/api/v1/ops/agent/heartbeat", _base_payload())
response = _post("/api/v1/ops/agent/heartbeat", _base_payload(), timeout=AGENT_HEARTBEAT_TIMEOUT_SECONDS)
_ensure_ok_response(response, "agent heartbeat failed")
def _pull_jobs() -> list[dict]:
response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE})
response = _post(
f"/api/v1/ops/agent/pull?limit=1",
{"node_code": NODE_CODE},
timeout=AGENT_PULL_TIMEOUT_SECONDS,
)
_ensure_ok_response(response, "agent pull failed")
data = response.get("data") or {}
return list(data.get("jobs") or [])
@@ -847,7 +1183,7 @@ def _job_complete(
path=f"/api/v1/ops/agent/jobs/{job_id}/complete",
payload=payload,
request_id=request_id,
timeout=30,
timeout=AGENT_JOB_COMPLETE_TIMEOUT_SECONDS,
)
@@ -883,7 +1219,7 @@ def _job_event(
path=f"/api/v1/ops/agent/jobs/{job_id}/events",
payload=delivery_payload,
request_id=request_id,
timeout=15,
timeout=AGENT_JOB_EVENT_TIMEOUT_SECONDS,
)
@@ -931,7 +1267,22 @@ def _process_job(job: dict) -> None:
"start_delivery_error": start_delivery_error,
},
)
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
try:
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
except Exception as exc:
ok = False
message = f"executor exception: {exc}"
result = {
"stdout": "",
"stderr": traceback.format_exc(),
"summary_text": message,
"exception_type": exc.__class__.__name__,
}
_log(
"job execute exception: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action} error={exc}"
)
stdout = str(result.get("stdout") or "")
stderr = str(result.get("stderr") or "")
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
@@ -956,6 +1307,57 @@ def _process_job(job: dict) -> None:
)
def _process_job_with_guard(job: dict) -> None:
normalized_job = _normalize_agent_job(job)
job_id = int(normalized_job.get("job_id") or 0)
action = str(normalized_job.get("action") or "").strip()
started_at = time.monotonic()
try:
_process_job(normalized_job)
except Exception as exc:
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
traceback_text = traceback.format_exc()
summary_text = f"node-agent fatal exception: {exc}"
_log(
"job fatal exception: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} error={exc}"
)
if job_id <= 0:
raise
try:
delivery = _job_complete(
job_id,
status="failed",
stdout="",
stderr=traceback_text,
result={
"stdout": "",
"stderr": traceback_text,
"summary_text": summary_text,
"exception_type": exc.__class__.__name__,
"fatal_loop_exception": True,
},
error_message=summary_text,
duration_ms=duration_ms,
summary_text=summary_text,
focus_ref=dict(normalized_job.get("focus_ref") or {}),
step_ref=dict(normalized_job.get("step_ref") or {}),
release_context=dict(normalized_job.get("release_context") or {}),
)
_log(
"job fatal exception completion: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} delivery={delivery.get('state')}"
)
except Exception as completion_exc:
_log(
"job fatal exception completion failed: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} error={completion_exc}"
)
def main() -> None:
if not NODE_CODE:
raise RuntimeError("NODE_CODE 未配置")
@@ -963,6 +1365,7 @@ def main() -> None:
_ensure_queue_dirs()
_register()
last_heartbeat_at = 0.0
last_runtime_config_sync_at = 0.0
while True:
now = time.time()
@@ -970,13 +1373,17 @@ def main() -> None:
delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT)
if delivery_summary["delivered"] or delivery_summary["dead_letter"]:
_log(f"delivery queue flush: {delivery_summary}")
if now - last_runtime_config_sync_at >= AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS:
bundle = _pull_runtime_config()
_apply_runtime_config(bundle)
last_runtime_config_sync_at = now
if now - last_heartbeat_at >= 15:
_heartbeat()
last_heartbeat_at = now
jobs = _pull_jobs()
if jobs:
for job in jobs:
_process_job(job)
_process_job_with_guard(job)
else:
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
except urllib.error.HTTPError as exc: