593 lines
24 KiB
Python
593 lines
24 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
|
|
|
|
def _format_time(value: datetime | None) -> str:
|
|
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
|
|
|
|
|
def _decode_json(value: object) -> dict:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if value in (None, ""):
|
|
return {}
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _normalize_region(value: str | None, fallback: str) -> str:
|
|
text = str(value or "").strip()
|
|
if not text or text == "unknown":
|
|
return str(fallback or "unknown").strip() or "unknown"
|
|
return text
|
|
|
|
|
|
def _should_append_runtime_projection(previous_payload: dict, current_projection: dict, previous_created_at: datetime | None) -> bool:
|
|
if not previous_payload:
|
|
return True
|
|
previous_projection = previous_payload.get("projection") or {}
|
|
if not previous_projection:
|
|
return True
|
|
|
|
keys_requiring_immediate_write = (
|
|
"worker_online",
|
|
"worker_mode",
|
|
"phase_label",
|
|
"phase_detail",
|
|
"proxy_runtime_label",
|
|
"proxy_runtime_reason",
|
|
)
|
|
for key in keys_requiring_immediate_write:
|
|
if previous_projection.get(key) != current_projection.get(key):
|
|
return True
|
|
|
|
previous_job = previous_projection.get("active_job") or {}
|
|
current_job = current_projection.get("active_job") or {}
|
|
for key in ("job_id", "job_code", "status"):
|
|
if previous_job.get(key) != current_job.get(key):
|
|
return True
|
|
|
|
previous_cluster = previous_projection.get("cluster_summary") or {}
|
|
current_cluster = current_projection.get("cluster_summary") or {}
|
|
for key in ("online_worker_nodes", "online_control_nodes", "busy_nodes", "stale_nodes", "offline_nodes"):
|
|
if previous_cluster.get(key) != current_cluster.get(key):
|
|
return True
|
|
|
|
previous_progress = previous_projection.get("progress") or {}
|
|
current_progress = current_projection.get("progress") or {}
|
|
failed_delta = abs(int(current_progress.get("failed", 0) or 0) - int(previous_progress.get("failed", 0) or 0))
|
|
blacklisted_delta = abs(int(current_progress.get("blacklisted", 0) or 0) - int(previous_progress.get("blacklisted", 0) or 0))
|
|
completed_delta = abs(int(current_progress.get("completed", 0) or 0) - int(previous_progress.get("completed", 0) or 0))
|
|
running_delta = abs(int(current_progress.get("running", 0) or 0) - int(previous_progress.get("running", 0) or 0))
|
|
if failed_delta > 0 or blacklisted_delta >= 10 or completed_delta >= 20 or running_delta >= 5:
|
|
return True
|
|
|
|
previous_alerts = previous_projection.get("dependency_alerts") or []
|
|
current_alerts = current_projection.get("dependency_alerts") or []
|
|
if previous_alerts != current_alerts:
|
|
return True
|
|
|
|
if not previous_created_at:
|
|
return True
|
|
now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now()
|
|
return now - previous_created_at >= timedelta(seconds=45)
|
|
|
|
|
|
def list_sync_records(limit: int = 20) -> list[dict]:
|
|
safe_limit = max(1, min(int(limit or 20), 200))
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(safe_limit,),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [
|
|
{
|
|
"id": row[0],
|
|
"sync_type": row[1],
|
|
"source_region": row[2],
|
|
"target_region": row[3],
|
|
"status": row[4],
|
|
"payload": _decode_json(row[5]),
|
|
"error_message": row[6] or "",
|
|
"created_at": _format_time(row[7]),
|
|
"updated_at": _format_time(row[8]),
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def _latest_sync_record_by_source(
|
|
cur,
|
|
*,
|
|
sync_type: str,
|
|
source_region: str,
|
|
target_region: str,
|
|
source_record_id: int,
|
|
) -> dict | None:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(sync_type, source_region, target_region, str(int(source_record_id))),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"sync_type": row[1],
|
|
"source_region": row[2],
|
|
"target_region": row[3],
|
|
"status": row[4],
|
|
"payload": _decode_json(row[5]),
|
|
"error_message": row[6] or "",
|
|
"created_at": _format_time(row[7]),
|
|
"updated_at": _format_time(row[8]),
|
|
}
|
|
|
|
|
|
def get_detect_result_sync_batches(limit: int = 5) -> dict:
|
|
safe_limit = max(1, min(int(limit or 5), 20))
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
batches: list[dict] = []
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, job_code, status, created_at, started_at, finished_at
|
|
FROM detect_jobs
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(safe_limit,),
|
|
)
|
|
jobs = cur.fetchall()
|
|
|
|
for row in jobs:
|
|
job_id = int(row[0])
|
|
job_code = str(row[1] or "")
|
|
job_status = str(row[2] or "")
|
|
created_at = _format_time(row[3])
|
|
started_at = _format_time(row[4])
|
|
finished_at = _format_time(row[5])
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT status, count(*)
|
|
FROM detect_job_items
|
|
WHERE job_id = %s
|
|
GROUP BY status
|
|
""",
|
|
(job_id,),
|
|
)
|
|
item_counts = {str(status or ""): int(count) for status, count in cur.fetchall()}
|
|
items_total = sum(item_counts.values())
|
|
items_terminal = int(item_counts.get("completed", 0)) + int(item_counts.get("blacklisted", 0)) + int(item_counts.get("failed", 0))
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT id, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_result_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->'projection'->'job'->>'job_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(source_region, target_region, str(job_id)),
|
|
)
|
|
projection_row = cur.fetchone()
|
|
projection = None
|
|
latest_push = None
|
|
latest_ingest = None
|
|
sync_state = "unsynced"
|
|
sync_message = "该任务还没有生成结果投影"
|
|
|
|
if projection_row:
|
|
projection_payload = _decode_json(projection_row[2])
|
|
projection = {
|
|
"id": int(projection_row[0]),
|
|
"status": projection_row[1],
|
|
"payload": projection_payload,
|
|
"created_at": _format_time(projection_row[3]),
|
|
"updated_at": _format_time(projection_row[4]),
|
|
}
|
|
latest_push = _latest_sync_record_by_source(
|
|
cur,
|
|
sync_type="runtime_push",
|
|
source_region=source_region,
|
|
target_region=target_region,
|
|
source_record_id=projection["id"],
|
|
)
|
|
latest_ingest = _latest_sync_record_by_source(
|
|
cur,
|
|
sync_type="detect_result_ingest",
|
|
source_region=source_region,
|
|
target_region=target_region,
|
|
source_record_id=projection["id"],
|
|
)
|
|
|
|
if latest_ingest:
|
|
sync_state = "synced"
|
|
sync_message = "最近一条结果投影已被目标地域接收"
|
|
elif latest_push and latest_push.get("status") == "success":
|
|
sync_state = "delivered"
|
|
sync_message = "结果投影已推送成功，等待目标侧回看接收记录"
|
|
elif latest_push and latest_push.get("status") == "pending":
|
|
sync_state = "pushing"
|
|
sync_message = "结果投影正在推送中"
|
|
elif latest_push and latest_push.get("status") == "failed":
|
|
sync_state = "failed"
|
|
sync_message = latest_push.get("error_message") or "最近一次结果投影推送失败"
|
|
else:
|
|
sync_state = "projected"
|
|
sync_message = "已生成结果投影，等待同步代理推送"
|
|
|
|
batches.append(
|
|
{
|
|
"job_id": job_id,
|
|
"job_code": job_code,
|
|
"job_status": job_status,
|
|
"created_at": created_at,
|
|
"started_at": started_at,
|
|
"finished_at": finished_at,
|
|
"items_total": items_total,
|
|
"items_terminal": items_terminal,
|
|
"items_pending": int(item_counts.get("pending", 0)),
|
|
"items_running": int(item_counts.get("running", 0)) + int(item_counts.get("claimed", 0)),
|
|
"items_failed": int(item_counts.get("failed", 0)),
|
|
"progress_percent": round((items_terminal / items_total) * 100, 2) if items_total else 0,
|
|
"sync_state": sync_state,
|
|
"sync_message": sync_message,
|
|
"projection": projection,
|
|
"latest_push": latest_push,
|
|
"latest_ingest": latest_ingest,
|
|
}
|
|
)
|
|
|
|
state_counts = {
|
|
"synced": 0,
|
|
"delivered": 0,
|
|
"pushing": 0,
|
|
"projected": 0,
|
|
"failed": 0,
|
|
"unsynced": 0,
|
|
}
|
|
for item in batches:
|
|
state = str(item.get("sync_state") or "unsynced")
|
|
state_counts[state] = state_counts.get(state, 0) + 1
|
|
|
|
return {
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
"jobs_total": len(batches),
|
|
"state_counts": state_counts,
|
|
"batches": batches,
|
|
}
|
|
|
|
|
|
def get_sync_summary(record_limit: int = 10) -> dict:
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT status, count(*)
|
|
FROM detect_sync_records
|
|
GROUP BY status
|
|
"""
|
|
)
|
|
status_counts = {str(status or "unknown"): int(count) for status, count in cur.fetchall()}
|
|
cur.execute(
|
|
"""
|
|
SELECT sync_type, count(*)
|
|
FROM detect_sync_records
|
|
GROUP BY sync_type
|
|
"""
|
|
)
|
|
type_counts = {str(sync_type or "unknown"): int(count) for sync_type, count in cur.fetchall()}
|
|
cur.execute("SELECT count(*) FROM detect_sync_records")
|
|
total = int(cur.fetchone()[0] or 0)
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
"""
|
|
)
|
|
latest = cur.fetchone()
|
|
|
|
latest_record = None
|
|
if latest:
|
|
latest_record = {
|
|
"id": latest[0],
|
|
"sync_type": latest[1],
|
|
"source_region": latest[2],
|
|
"target_region": latest[3],
|
|
"status": latest[4],
|
|
"payload": _decode_json(latest[5]),
|
|
"error_message": latest[6] or "",
|
|
"created_at": _format_time(latest[7]),
|
|
"updated_at": _format_time(latest[8]),
|
|
}
|
|
|
|
return {
|
|
"enabled": bool(settings.sync_push_enabled),
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
"target_api_base_url": settings.sync_target_api_base_url,
|
|
"batch_size": max(1, int(settings.sync_batch_size or 200)),
|
|
"poll_interval_seconds": max(5, int(settings.sync_poll_interval_seconds or 30)),
|
|
"records_total": total,
|
|
"status_counts": status_counts,
|
|
"type_counts": type_counts,
|
|
"latest_record": latest_record,
|
|
"detect_result_batches": get_detect_result_sync_batches(limit=min(5, record_limit)),
|
|
"recent_records": list_sync_records(limit=record_limit),
|
|
}
|
|
|
|
|
|
def append_sync_record(
|
|
*,
|
|
sync_type: str,
|
|
source_region: str,
|
|
target_region: str,
|
|
status: str,
|
|
payload: dict | None = None,
|
|
error_message: str = "",
|
|
) -> int:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
str(sync_type or "").strip() or "unknown",
|
|
_normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)),
|
|
_normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")),
|
|
str(status or "").strip() or "pending",
|
|
json.dumps(payload or {}, ensure_ascii=False),
|
|
str(error_message or "").strip(),
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def append_runtime_projection_if_changed(
|
|
*,
|
|
detect: dict,
|
|
cluster: dict,
|
|
source_region: str | None = None,
|
|
target_region: str | None = None,
|
|
) -> int | None:
|
|
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
|
|
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
|
|
active_job = detect.get("active_job") or {}
|
|
projection = {
|
|
"worker_online": bool(detect.get("worker_online", False)),
|
|
"worker_mode": detect.get("worker_mode", ""),
|
|
"phase_label": detect.get("phase_label", ""),
|
|
"phase_detail": detect.get("phase_detail", ""),
|
|
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
|
|
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
|
|
"progress": {
|
|
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
|
|
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
|
|
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
|
|
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
|
|
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
|
|
},
|
|
"active_job": {
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
"items_total": active_job.get("items_total", 0),
|
|
"items_terminal": active_job.get("items_terminal", 0),
|
|
"items_pending": active_job.get("items_pending", 0),
|
|
"items_running": active_job.get("items_running", 0),
|
|
"items_failed": active_job.get("items_failed", 0),
|
|
},
|
|
"cluster_summary": {
|
|
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
|
|
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
|
|
"online_control_nodes": int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0),
|
|
"busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []),
|
|
"stale_nodes": list((cluster.get("summary") or {}).get("stale_nodes") or []),
|
|
"offline_nodes": list((cluster.get("summary") or {}).get("offline_nodes") or []),
|
|
},
|
|
"dependency_alerts": [
|
|
{
|
|
"kind": item.get("kind", ""),
|
|
"title": item.get("title", ""),
|
|
"level": item.get("level", ""),
|
|
}
|
|
for item in (detect.get("dependency_alerts") or [])[:3]
|
|
],
|
|
}
|
|
payload = {
|
|
"projection": projection,
|
|
"projection_hash": hashlib.sha1(
|
|
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
).hexdigest(),
|
|
}
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT payload_json, created_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'runtime_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(normalized_source_region, normalized_target_region),
|
|
)
|
|
latest = cur.fetchone()
|
|
latest_payload = _decode_json(latest[0]) if latest else {}
|
|
latest_created_at = latest[1] if latest else None
|
|
if latest_payload.get("projection_hash") == payload["projection_hash"]:
|
|
return None
|
|
if not _should_append_runtime_projection(latest_payload, projection, latest_created_at):
|
|
return None
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"runtime_projection",
|
|
normalized_source_region,
|
|
normalized_target_region,
|
|
"projected",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def append_detect_result_projection_if_changed(
|
|
*,
|
|
detect: dict,
|
|
source_region: str | None = None,
|
|
target_region: str | None = None,
|
|
) -> int | None:
|
|
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
|
|
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
|
|
active_job = detect.get("active_job") or {}
|
|
if not active_job:
|
|
return None
|
|
|
|
latest_cycle_event = active_job.get("latest_cycle_event") or active_job.get("latest_event") or {}
|
|
projection = {
|
|
"job": {
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
"items_total": active_job.get("items_total", 0),
|
|
"items_pending": active_job.get("items_pending", 0),
|
|
"items_claimed": active_job.get("items_claimed", 0),
|
|
"items_running": active_job.get("items_running", 0),
|
|
"items_completed": active_job.get("items_completed", 0),
|
|
"items_blacklisted": active_job.get("items_blacklisted", 0),
|
|
"items_failed": active_job.get("items_failed", 0),
|
|
"items_terminal": active_job.get("items_terminal", 0),
|
|
"current_cycle_token": active_job.get("current_cycle_token", ""),
|
|
},
|
|
"latest_event": {
|
|
"node_code": latest_cycle_event.get("node_code", ""),
|
|
"event_type": latest_cycle_event.get("event_type", ""),
|
|
"message": latest_cycle_event.get("message", ""),
|
|
"created_at": latest_cycle_event.get("created_at", ""),
|
|
},
|
|
"queue": {
|
|
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
|
|
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
|
|
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
|
|
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
|
|
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
|
|
},
|
|
"phase": {
|
|
"label": detect.get("phase_label", ""),
|
|
"detail": detect.get("phase_detail", ""),
|
|
},
|
|
}
|
|
payload = {
|
|
"projection": projection,
|
|
"projection_hash": hashlib.sha1(
|
|
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
).hexdigest(),
|
|
}
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT payload_json, created_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_result_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(normalized_source_region, normalized_target_region),
|
|
)
|
|
latest = cur.fetchone()
|
|
latest_payload = _decode_json(latest[0]) if latest else {}
|
|
latest_created_at = latest[1] if latest else None
|
|
if latest_payload.get("projection_hash") == payload["projection_hash"]:
|
|
return None
|
|
|
|
latest_projection = latest_payload.get("projection") or {}
|
|
latest_job = latest_projection.get("job") or {}
|
|
current_job = projection.get("job") or {}
|
|
latest_event = latest_projection.get("latest_event") or {}
|
|
current_event = projection.get("latest_event") or {}
|
|
if latest_job.get("status") == current_job.get("status") and latest_event == current_event and latest_created_at:
|
|
now = datetime.now(latest_created_at.tzinfo) if latest_created_at.tzinfo else datetime.now()
|
|
if now - latest_created_at < timedelta(seconds=30):
|
|
return None
|
|
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"detect_result_projection",
|
|
normalized_source_region,
|
|
normalized_target_region,
|
|
"projected",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|