admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ingest mainland detect result events

Browse Source
This commit is contained in:
Your Name
2026-04-19 02:53:43 +08:00
parent 3a55db94f8
commit 4dcbf8ecb3
7 changed files with 600 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
domain-api/app/services/sync_push_service.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import json
import socket
import urllib.error
@@ -14,6 +15,14 @@ from app.services.cluster_runtime_service import cleanup_imported_runtime_nodes,
from app.services.sync_record_service import _decode_json, _normalize_region
_DETECT_RESULT_EVENT_TYPES = {
"domain_started",
"domain_completed",
"domain_failed",
"domain_blacklisted",
}
def _format_time(value: datetime | None) -> str:
return value.isoformat(sep=" ", timespec="seconds") if value else ""
@@ -664,6 +673,173 @@ def _update_push_attempt(record_id: int, *, status: str, payload: dict | None =
conn.commit()
def _build_detect_result_import_fingerprint(
*,
source_region: str,
source_record_id: int,
source_job_code: str,
event: dict,
) -> str:
payload = _decode_json(event.get("payload"))
seed = {
"source_region": str(source_region or "").strip(),
"source_record_id": int(source_record_id or 0),
"source_job_code": str(source_job_code or "").strip(),
"node_code": str(event.get("node_code") or "").strip(),
"event_type": str(event.get("event_type") or "").strip(),
"message": str(event.get("message") or "").strip(),
"created_at": str(event.get("created_at") or "").strip(),
"domain": str(payload.get("domain") or "").strip(),
"domain_id": str(payload.get("domain_id") or "").strip(),
"status": str(payload.get("status") or "").strip(),
"cycle_token": str(payload.get("cycle_token") or "").strip(),
}
return hashlib.sha1(json.dumps(seed, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest()
def _extract_detect_result_projection_events(
*,
source_region: str,
source_record_id: int,
projection: dict,
) -> list[dict]:
source_job = projection.get("job") or {}
source_job_code = str(source_job.get("job_code") or "").strip()
source_job_id = int(source_job.get("job_id") or 0)
events: list[dict] = []
for raw_event in list(projection.get("recent_domain_events") or []):
event_type = str(raw_event.get("event_type") or "").strip()
if event_type not in _DETECT_RESULT_EVENT_TYPES:
continue
payload = _decode_json(raw_event.get("payload"))
fingerprint = _build_detect_result_import_fingerprint(
source_region=source_region,
source_record_id=source_record_id,
source_job_code=source_job_code,
event=raw_event,
)
payload.update(
{
"imported_from_projection": True,
"import_source_region": source_region,
"import_source_record_id": int(source_record_id or 0),
"import_source_job_id": source_job_id,
"import_source_job_code": source_job_code,
"import_fingerprint": fingerprint,
}
)
events.append(
{
"node_code": str(raw_event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(raw_event.get("level") or "info").strip() or "info",
"message": str(raw_event.get("message") or "").strip(),
"created_at": str(raw_event.get("created_at") or "").strip(),
"payload": payload,
"fingerprint": fingerprint,
}
)
return events
def _resolve_detect_result_target_job_id() -> int:
from app.services.detect_job_service import get_active_detect_job_summary
active_job = get_active_detect_job_summary(event_limit=1) or {}
return int(active_job.get("job_id") or 0)
def _parse_event_created_at(value: str) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except Exception:
return None
def _import_detect_result_projection_events(
*,
source_region: str,
source_record_id: int,
projection: dict,
) -> dict:
events = _extract_detect_result_projection_events(
source_region=source_region,
source_record_id=source_record_id,
projection=projection,
)
if not events:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
target_job_id = _resolve_detect_result_target_job_id()
if target_job_id <= 0:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
imported_count = 0
deduplicated_count = 0
with get_db() as conn:
with conn.cursor() as cur:
for event in events:
cur.execute(
"""
SELECT id
FROM detect_run_events
WHERE job_id = %s
AND (payload_json->>'import_fingerprint') = %s
ORDER BY id DESC
LIMIT 1
""",
(target_job_id, event["fingerprint"]),
)
if cur.fetchone():
deduplicated_count += 1
continue
created_at = _parse_event_created_at(event.get("created_at", ""))
if created_at:
cur.execute(
"""
INSERT INTO detect_run_events (
job_id, node_code, event_type, level, message, payload_json, created_at
) VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s)
""",
(
target_job_id,
event["node_code"],
event["event_type"],
event["level"],
event["message"],
json.dumps(event["payload"], ensure_ascii=False),
created_at,
),
)
else:
cur.execute(
"""
INSERT INTO detect_run_events (
job_id, node_code, event_type, level, message, payload_json
) VALUES (%s, %s, %s, %s, %s, %s::jsonb)
""",
(
target_job_id,
event["node_code"],
event["event_type"],
event["level"],
event["message"],
json.dumps(event["payload"], ensure_ascii=False),
),
)
imported_count += 1
conn.commit()
return {
"imported_count": imported_count,
"deduplicated_count": deduplicated_count,
"target_job_id": target_job_id,
}
def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
configured_token = str(settings.sync_shared_token or "").strip()
incoming_token = str(shared_token or "").strip()
@@ -699,6 +875,14 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
if existing:
if sync_type == "runtime_projection":
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
if sync_type == "detect_result_projection":
import_result = _import_detect_result_projection_events(
source_region=source_region,
source_record_id=source_record_id,
projection=projection,
)
else:
import_result = {}
cur.execute(
"""
UPDATE detect_sync_records
@@ -708,7 +892,11 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
(int(existing[0]),),
)
conn.commit()
return True, "同步投影已存在，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
return True, "同步投影已存在，已按幂等处理", {
"record_id": int(existing[0]),
"deduplicated": True,
"event_import": import_result,
}
stored_payload = {
"sync_type": sync_type,
@@ -737,7 +925,19 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
conn.commit()
if sync_type == "runtime_projection":
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
return True, "同步投影接收成功", {"record_id": record_id, "deduplicated": False}
if sync_type == "detect_result_projection":
import_result = _import_detect_result_projection_events(
source_region=source_region,
source_record_id=source_record_id,
projection=projection,
)
else:
import_result = {}
return True, "同步投影接收成功", {
"record_id": record_id,
"deduplicated": False,
"event_import": import_result,
}
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:

32
domain-api/app/services/sync_record_service.py
View File

@@ -38,6 +38,37 @@ def _normalize_region(value: str | None, fallback: str) -> str:
return text
_DETECT_RESULT_EVENT_TYPES = {
"domain_started",
"domain_completed",
"domain_failed",
"domain_blacklisted",
}
def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]:
events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
normalized: list[dict] = []
for event in reversed(events):
event_type = str(event.get("event_type") or "").strip()
if event_type not in _DETECT_RESULT_EVENT_TYPES:
continue
payload = _decode_json(event.get("payload"))
normalized.append(
{
"node_code": str(event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(event.get("level") or "info").strip() or "info",
"message": str(event.get("message") or "").strip(),
"created_at": str(event.get("created_at") or "").strip(),
"payload": payload,
}
)
if limit <= 0:
return normalized
return normalized[-int(limit):]
def _build_detect_result_batch_digest(batch: dict | None) -> dict:
batch = batch or {}
projection = (batch.get("projection") or {}).get("payload") or {}
@@ -608,6 +639,7 @@ def append_detect_result_projection_if_changed(
"label": detect.get("phase_label", ""),
"detail": detect.get("phase_detail", ""),
},
"recent_domain_events": _collect_recent_domain_events(active_job, limit=30),
}
payload = {
"projection": projection,