admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
domain-api/app/services/ops_agent_service.py
View File

@@ -12,6 +12,9 @@ from app.core.db import get_db
from app.services.ops_command_service import build_bash_command
from app.services.ops_job_service import create_ops_job, ensure_ops_schema, get_ops_job
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
from app.services.runtime_settings_service import get_runtime_settings
from app.services.sensitive_words_service import get_sensitive_words_payload
from app.services.settings_service import get_settings_payload
_AGENT_SCHEMA_SQL = """
@@ -117,6 +120,48 @@ def _format_time(value: object) -> str:
return ""
def _prefer_non_loopback_identity(primary: object, fallback: object) -> str:
primary_value = str(primary or "").strip()
fallback_value = str(fallback or "").strip()
invalid_values = {"", "localhost", "localhost.localdomain", "127.0.0.1", "::1"}
if primary_value and primary_value.lower() not in invalid_values:
return primary_value
return fallback_value
def _merge_detect_runtime_snapshot(cluster_metadata: dict, metadata: dict, current_load: int) -> dict:
active_threads = int(cluster_metadata.get("active_threads", metadata.get("active_threads", 0)) or 0)
max_threads = int(cluster_metadata.get("max_threads", metadata.get("max_threads", 0)) or 0)
inferred_worker_online = bool(
cluster_metadata.get("worker_online", metadata.get("worker_online", False))
or cluster_metadata.get("service_running", metadata.get("service_running", False))
or current_load > 0
or active_threads > 0
)
inferred_detect_participating = bool(
cluster_metadata.get("detect_participating", metadata.get("detect_participating", False))
or current_load > 0
or active_threads > 0
)
return {
"worker_online": inferred_worker_online,
"detect_participating": inferred_detect_participating,
"active_threads": active_threads,
"max_threads": max_threads,
"current_load": current_load,
"phase_label": str(cluster_metadata.get("phase_label", metadata.get("phase_label", "")) or "").strip(),
"phase_detail": str(cluster_metadata.get("phase_detail", metadata.get("phase_detail", "")) or "").strip(),
"recent_warning": str(cluster_metadata.get("recent_warning", metadata.get("recent_warning", "")) or "").strip(),
"proxy_runtime_label": str(
cluster_metadata.get("proxy_runtime_label", metadata.get("proxy_runtime_label", "")) or ""
).strip(),
"proxy_runtime_reason": str(
cluster_metadata.get("proxy_runtime_reason", metadata.get("proxy_runtime_reason", "")) or ""
).strip(),
"updated_at": str(cluster_metadata.get("updated_at", metadata.get("updated_at", "")) or "").strip(),
}
def _ops_job_event_level_label(level: object) -> str:
normalized_level = str(level or "").strip().lower()
mapping = {
@@ -588,6 +633,7 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
seen_node_codes.add(node_code)
metadata = dict(node.get("metadata") or {})
cluster_node = cluster_map.get(node_code, {})
cluster_metadata = dict(cluster_node.get("metadata") or {})
latest_token = latest_tokens.get(node_code, {})
latest_job = latest_jobs.get(node_code, {})
participation_row = dict(detect_participation_map.get(node_code) or {})
@@ -607,6 +653,15 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
current_load=current_load,
)
capabilities = list(metadata.get("capabilities") or [])
detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load)
active_job_snapshot = {
"job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(),
"status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(),
"items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0),
"items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0),
"items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0),
"items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0),
}
merged_node = {
**node,
"is_managed": True,
@@ -620,10 +675,14 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
"agent_version": str(metadata.get("agent_version") or "").strip(),
"agent_hostname": str(metadata.get("hostname") or "").strip(),
"agent_ip": str(metadata.get("ip") or "").strip(),
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
"cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_status": cluster_status,
"cluster_current_load": current_load,
"current_load": current_load,
"detect_runtime": detect_runtime,
"runtime_state": detect_runtime,
"active_job": active_job_snapshot,
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or metadata.get("last_heartbeat_at") or "").strip(),
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", metadata.get("is_effective_worker", False))),
"cluster_detect_participating": bool(
@@ -688,6 +747,7 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
if not node_code or node_code in seen_node_codes:
continue
metadata = dict(cluster_node.get("metadata") or {})
cluster_metadata = dict(cluster_node.get("metadata") or {})
latest_token = latest_tokens.get(node_code, {})
latest_job = latest_jobs.get(node_code, {})
participation_row = dict(detect_participation_map.get(node_code) or {})
@@ -706,6 +766,15 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
current_load=current_load,
)
capabilities = list(metadata.get("capabilities") or [])
detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load)
active_job_snapshot = {
"job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(),
"status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(),
"items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0),
"items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0),
"items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0),
"items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0),
}
fallback_node = {
"node_code": node_code,
"region": str(cluster_node.get("region") or "").strip(),
@@ -732,12 +801,16 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
"capabilities": capabilities,
"capabilities_count": len(capabilities),
"agent_version": str(metadata.get("agent_version") or "").strip(),
"agent_hostname": str(cluster_node.get("hostname") or metadata.get("hostname") or "").strip(),
"agent_ip": str(cluster_node.get("ip") or metadata.get("ip") or "").strip(),
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
"agent_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"agent_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_status": cluster_status,
"cluster_current_load": current_load,
"current_load": current_load,
"detect_runtime": detect_runtime,
"runtime_state": detect_runtime,
"active_job": active_job_snapshot,
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or "").strip(),
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", False)),
"cluster_detect_participating": bool(
@@ -2359,7 +2432,7 @@ def build_node_agent_bootstrap_plan(
f"SYNC_AGENT_SERVICE_NAME={settings.sync_agent_service_name}",
"NODE_AGENT_SERVICE_NAME=domaincheck-node-agent",
"",
'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]',
'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.start_detection","runtime.stop_detection","runtime.pull_tasks","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","runtime.reset_lab_state","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]',
"OPS_AGENT_LABELS={}",
]
env_content = "\n".join(env_lines)
@@ -2529,6 +2602,135 @@ def _upsert_agent_runtime(node_code: str, payload: dict) -> None:
conn.commit()
def _load_existing_detect_node_runtime(node_code: str) -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return {}
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT status, current_load, metadata_json, last_heartbeat_at
FROM detect_worker_nodes
WHERE node_code = %s
LIMIT 1
""",
(normalized_node_code,),
)
row = cur.fetchone()
except Exception:
return {}
if not row:
return {}
metadata = row[2] if isinstance(row[2], dict) else {}
return {
"status": str(row[0] or "").strip(),
"current_load": int(row[1] or 0),
"metadata": metadata,
"last_heartbeat_at": row[3],
}
def _upsert_agent_detect_runtime(node_code: str, payload: dict) -> None:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return
metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}
detect_runtime = metadata.get("detect_runtime") if isinstance(metadata.get("detect_runtime"), dict) else {}
if not detect_runtime:
return
try:
from app.services.cluster_runtime_service import register_node_heartbeat
region = str(payload.get("region") or "unknown").strip() or "unknown"
role = str(payload.get("role") or "worker").strip() or "worker"
active_threads = max(0, int(detect_runtime.get("active_threads") or 0))
max_threads = max(0, int(detect_runtime.get("max_threads") or 0))
current_load = max(
0,
int(detect_runtime.get("current_load") or 0),
active_threads,
)
worker_online = bool(detect_runtime.get("worker_online", False) or detect_runtime.get("service_running", False))
detect_participating = bool(detect_runtime.get("detect_participating", False) or current_load > 0)
phase_label = str(detect_runtime.get("phase_label") or "").strip()
phase_detail = str(detect_runtime.get("phase_detail") or "").strip()
recent_warning = str(detect_runtime.get("recent_warning") or "").strip()
has_runtime_signal = any(
[
worker_online,
detect_participating,
current_load > 0,
active_threads > 0,
max_threads > 0,
bool(phase_label),
bool(phase_detail),
bool(recent_warning),
]
)
if not has_runtime_signal:
return
existing_runtime = _load_existing_detect_node_runtime(normalized_node_code)
existing_metadata = existing_runtime.get("metadata") if isinstance(existing_runtime.get("metadata"), dict) else {}
existing_active_threads = max(0, int(existing_metadata.get("active_threads", 0) or 0))
existing_current_load = max(0, int(existing_runtime.get("current_load", 0) or 0), existing_active_threads)
existing_status = str(existing_runtime.get("status") or "").strip()
existing_last_heartbeat_at = existing_runtime.get("last_heartbeat_at")
existing_is_recent = False
if isinstance(existing_last_heartbeat_at, datetime):
current_time = (
datetime.now(existing_last_heartbeat_at.tzinfo)
if existing_last_heartbeat_at.tzinfo
else datetime.now()
)
existing_is_recent = (current_time - existing_last_heartbeat_at) <= timedelta(seconds=120)
normalized_phase_detail = phase_detail.lower()
generic_phase_detail = bool(normalized_phase_detail) and "/" in normalized_phase_detail and " " not in normalized_phase_detail
weak_agent_snapshot = bool(
worker_online
and current_load <= 0
and active_threads <= 0
and max_threads > 0
and not phase_label
and (not phase_detail or generic_phase_detail)
and not recent_warning
)
if weak_agent_snapshot and existing_is_recent and (
existing_current_load > 0 or existing_status == "busy"
):
return
status = "busy" if current_load > 0 else ("online" if worker_online else "unknown")
heartbeat_metadata = {
"service": "agent-heartbeat",
"worker_online": worker_online,
"detect_participating": detect_participating,
"active_threads": active_threads,
"max_threads": max_threads,
"phase_label": phase_label,
"phase_detail": phase_detail,
"recent_warning": recent_warning,
"updated_at": str(detect_runtime.get("updated_at") or "").strip(),
"agent_heartbeat_at": datetime.now().isoformat(timespec="seconds"),
}
register_node_heartbeat(
node_code=normalized_node_code,
region=region,
role=role,
status=status,
current_load=current_load,
metadata=heartbeat_metadata,
hostname_override=str(payload.get("hostname") or "").strip(),
ip_override=str(payload.get("ip") or "").strip(),
)
except Exception:
return
def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
@@ -2537,6 +2739,7 @@ def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]:
if not ok:
return False, message, auth
_upsert_agent_runtime(node_code, payload)
_upsert_agent_detect_runtime(node_code, payload)
return True, "Agent 注册成功", {
"node_code": node_code,
"expires_at": auth.get("expires_at", ""),
@@ -2552,6 +2755,7 @@ def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]:
if not ok:
return False, message, auth
_upsert_agent_runtime(node_code, payload)
_upsert_agent_detect_runtime(node_code, payload)
return True, "heartbeat ok", {
"node_code": node_code,
"server_time": _format_time(datetime.now()),
@@ -2559,6 +2763,57 @@ def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]:
}
def _runtime_config_bundle_hash_payload(bundle: dict | None) -> dict:
normalized = dict(bundle or {})
normalized.pop("config_hash", None)
normalized.pop("generated_at", None)
return normalized
def _build_agent_runtime_config_bundle(node_code: str) -> dict:
settings_payload = get_settings_payload()
runtime_settings = get_runtime_settings()
sensitive_words_payload = get_sensitive_words_payload()
bundle = {
"node_code": str(node_code or "").strip(),
"detect_options": dict(settings_payload.get("detect_options") or {}),
"proxy_config": dict(settings_payload.get("proxy_config") or {}),
"thread_count": int(settings_payload.get("thread_count", 2) or 2),
"node_thread_counts": dict(settings_payload.get("node_thread_counts") or {}),
"runtime_settings": dict(runtime_settings or {}),
"sensitive_words": {
"text": str(sensitive_words_payload.get("text") or ""),
"total": int(sensitive_words_payload.get("total", 0) or 0),
"items": list(sensitive_words_payload.get("items") or []),
},
"generated_at": _format_time(datetime.now()),
}
bundle["config_hash"] = hashlib.sha256(
json.dumps(
_runtime_config_bundle_hash_payload(bundle),
ensure_ascii=False,
sort_keys=True,
).encode("utf-8")
).hexdigest()
return bundle
def agent_pull_runtime_config(payload: dict, *, token: str) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
return _agent_error("node_code 不能为空", "agent_node_code_required")
ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code)
if not ok:
return False, message, auth
bundle = _build_agent_runtime_config_bundle(node_code)
return True, "runtime config ok", {
"node_code": node_code,
"server_time": _format_time(datetime.now()),
"expires_at": auth.get("expires_at", ""),
"bundle": bundle,
}
def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
@@ -2568,6 +2823,7 @@ def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool,
return False, message, _auth
safe_limit = min(max(int(limit or 1), 1), 10)
dispatched_events: list[dict] = []
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2604,15 +2860,19 @@ def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool,
""",
(job_id,),
)
append_ops_job_event(
job_id=job_id,
node_code=node_code,
event_type="agent_dispatched",
message=f"任务已派发给节点 {node_code}",
payload={"node_code": node_code},
dispatched_events.append(
{
"job_id": job_id,
"node_code": node_code,
"event_type": "agent_dispatched",
"message": f"任务已派发给节点 {node_code}",
"payload": {"node_code": node_code},
}
)
jobs.append(_agent_job_envelope(get_ops_job(job_id)))
conn.commit()
for event in dispatched_events:
append_ops_job_event(**event)
return True, "ok", {
"jobs": jobs,
"count": len(jobs),
@@ -2631,6 +2891,7 @@ def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[b
if not ok:
return False, message, _auth
started_event: dict | None = None
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2662,14 +2923,16 @@ def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[b
)
step_rows = cur.fetchall()
step_ids = [int(item[0]) for item in step_rows]
append_ops_job_event(
job_id=int(job_id),
node_code=node_code,
event_type="agent_started",
message=f"节点 {node_code} 已开始执行任务",
payload={"step_ids": step_ids},
)
started_event = {
"job_id": int(job_id),
"node_code": node_code,
"event_type": "agent_started",
"message": f"节点 {node_code} 已开始执行任务",
"payload": {"step_ids": step_ids},
}
conn.commit()
if started_event:
append_ops_job_event(**started_event)
job = get_ops_job(int(job_id))
return True, "任务已标记为运行中", {
"job": job,
@@ -2710,6 +2973,7 @@ def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool,
result["focus_ref"] = focus_ref
event_level = "info" if job_status == "success" else ("warning" if job_status == "partially_succeeded" else "error")
completed_event: dict | None = None
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2789,22 +3053,24 @@ def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool,
)
step_rows = cur.fetchall()
step_ids = [int(item[0]) for item in step_rows]
append_ops_job_event(
job_id=int(job_id),
node_code=node_code,
client_event_id=(f"complete:{client_request_id}" if client_request_id else ""),
event_type="agent_completed",
message=f"节点 {node_code} 已完成任务，状态: {job_status}",
level=event_level,
payload={
completed_event = {
"job_id": int(job_id),
"node_code": node_code,
"client_event_id": (f"complete:{client_request_id}" if client_request_id else ""),
"event_type": "agent_completed",
"message": f"节点 {node_code} 已完成任务，状态: {job_status}",
"level": event_level,
"payload": {
"step_ids": step_ids,
"result": result,
"duration_ms": duration_ms,
"summary_text": str(result.get("summary_text") or result.get("summary") or summary_text).strip(),
"focus_ref": focus_ref,
},
)
}
conn.commit()
if completed_event:
append_ops_job_event(**completed_event)
from app.services.ops_release_service import refresh_release_rollout_for_job
refresh_release_rollout_for_job(int(job_id))