feat: add ops center and node onboarding flow

This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions

View File

@@ -0,0 +1,633 @@
from __future__ import annotations
from fastapi import APIRouter
from app.schemas.common import ApiResponse
from app.services.ops_agent_service import (
build_managed_node_handover_bootstrap_plan,
execute_managed_node_onboarding_bootstrap,
execute_managed_node_onboarding_acceptance,
execute_managed_node_onboarding_recovery,
get_managed_node_delivery_queue,
get_managed_node_handover,
get_managed_node_onboarding,
list_managed_node_delivery_queue_records,
list_managed_nodes_with_agent_state,
list_ops_job_events,
preview_managed_node_onboarding_bootstrap,
preview_managed_node_onboarding_acceptance,
preview_managed_node_onboarding_recovery,
request_managed_node_delivery_queue_action,
summarize_ops_job_events,
)
from app.services.ops_job_service import (
approve_ops_job,
cancel_ops_job,
create_ops_job,
create_ops_job_batch,
dispatch_ops_job,
get_ops_job,
list_ops_jobs,
sync_managed_nodes_from_cluster,
upsert_managed_node,
)
from app.services.ops_playbook_service import (
cancel_ops_playbook_run,
execute_ops_playbook,
get_ops_playbook_run,
get_ops_playbooks,
get_recent_ops_playbook_runs,
list_ops_playbook_run_events,
preview_ops_playbook,
rerun_ops_playbook_run,
)
from app.services.ops_policy_service import preview_ops_job_policy
from app.services.ops_service import (
execute_codex_action,
execute_resolved_driver_action,
get_ops_activity_stream,
get_ops_codex_brief,
get_ops_contract_detail,
get_ops_contract_registry,
get_ops_doctor_decision,
get_ops_driver_feed,
get_ops_go_live_summary,
get_ops_go_live_signoff,
get_ops_go_live_bundle,
get_ops_go_live_review,
get_ops_link_snapshot,
get_ops_node_scene_log,
get_ops_stack_diagnosis,
execute_driver_action,
preview_driver_action,
resolve_codex_action,
resolve_driver_action_request,
resolve_ops_runbook_sequence,
execute_ops_runbook_sequence,
get_ops_blueprint,
get_ops_capabilities,
get_ops_inspection_overview,
get_ops_overview,
get_ops_runbook,
)
from app.services.ops_template_service import get_ops_action_templates
router = APIRouter(tags=["ops"])
@router.get("/ops/overview", response_model=ApiResponse)
def ops_overview() -> ApiResponse:
return ApiResponse(data=get_ops_overview())
@router.get("/ops/link-snapshot", response_model=ApiResponse)
def ops_link_snapshot() -> ApiResponse:
return ApiResponse(data=get_ops_link_snapshot())
@router.get("/ops/stack-diagnosis", response_model=ApiResponse)
def ops_stack_diagnosis(base_url: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_stack_diagnosis(base_url=base_url))
@router.get("/ops/go-live-summary", response_model=ApiResponse)
def ops_go_live_summary(base_url: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_go_live_summary(base_url=base_url))
@router.get("/ops/doctor-decision", response_model=ApiResponse)
def ops_doctor_decision(base_url: str = "", report_dir: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_doctor_decision(base_url=base_url, report_dir=report_dir))
@router.get("/ops/go-live-signoff", response_model=ApiResponse)
def ops_go_live_signoff(base_url: str = "", report_dir: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_go_live_signoff(base_url=base_url, report_dir=report_dir))
@router.get("/ops/go-live-bundle", response_model=ApiResponse)
def ops_go_live_bundle(base_url: str = "", report_dir: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_go_live_bundle(base_url=base_url, report_dir=report_dir))
@router.get("/ops/go-live-review", response_model=ApiResponse)
def ops_go_live_review(base_url: str = "", report_dir: str = "") -> ApiResponse:
return ApiResponse(data=get_ops_go_live_review(base_url=base_url, report_dir=report_dir))
@router.get("/ops/inspection-overview", response_model=ApiResponse)
def ops_inspection_overview(
limit: int = 80,
status: str = "",
problem_kind: str = "",
query: str = "",
only_problem: bool = False,
only_participating: bool = False,
) -> ApiResponse:
return ApiResponse(
data=get_ops_inspection_overview(
fetch_limit=limit,
status=status,
problem_kind=problem_kind,
query=query,
only_problem=only_problem,
only_participating=only_participating,
)
)
@router.get("/ops/activity-stream", response_model=ApiResponse)
def ops_activity_stream(
limit: int = 18,
kind: str = "",
status: str = "",
execution_mode: str = "",
query: str = "",
) -> ApiResponse:
return ApiResponse(
data=get_ops_activity_stream(
limit=limit,
kind=kind,
status=status,
execution_mode=execution_mode,
query=query,
)
)
@router.get("/ops/driver-feed", response_model=ApiResponse)
def ops_driver_feed() -> ApiResponse:
return ApiResponse(data=get_ops_driver_feed())
@router.get("/ops/codex-brief", response_model=ApiResponse)
def ops_codex_brief() -> ApiResponse:
return ApiResponse(data=get_ops_codex_brief())
@router.get("/ops/capabilities", response_model=ApiResponse)
def ops_capabilities() -> ApiResponse:
return ApiResponse(data=get_ops_capabilities())
@router.get("/ops/contracts", response_model=ApiResponse)
def ops_contracts() -> ApiResponse:
return ApiResponse(data=get_ops_contract_registry())
@router.get("/ops/contracts/{contract_key}", response_model=ApiResponse)
def ops_contract_detail(contract_key: str) -> ApiResponse:
data = get_ops_contract_detail(contract_key)
if not data:
return ApiResponse(code=1, message="contract 不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/action-templates", response_model=ApiResponse)
def ops_action_templates() -> ApiResponse:
return ApiResponse(data=get_ops_action_templates())
@router.get("/ops/playbooks", response_model=ApiResponse)
def ops_playbooks() -> ApiResponse:
return ApiResponse(data=get_ops_playbooks())
@router.get("/ops/playbook-runs", response_model=ApiResponse)
def ops_playbook_runs(
limit: int = 12,
status: str = "",
group_key: str = "",
query: str = "",
) -> ApiResponse:
return ApiResponse(
data=get_recent_ops_playbook_runs(
limit=limit,
status=status,
group_key=group_key,
query=query,
)
)
@router.get("/ops/playbook-runs/{run_code}", response_model=ApiResponse)
def ops_playbook_run_detail(run_code: str) -> ApiResponse:
data = get_ops_playbook_run(run_code)
if not data:
return ApiResponse(code=1, message="playbook run 不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/playbook-runs/{run_code}/events", response_model=ApiResponse)
def ops_playbook_run_events(
run_code: str,
limit: int = 80,
step_key: str = "",
node_code: str = "",
) -> ApiResponse:
data = list_ops_playbook_run_events(
run_code,
limit=limit,
step_key=step_key,
node_code=node_code,
)
return ApiResponse(data=data)
@router.post("/ops/playbook-runs/{run_code}/rerun", response_model=ApiResponse)
def ops_playbook_run_rerun(run_code: str, payload: dict | None = None) -> ApiResponse:
payload = payload or {}
ok, message, data = rerun_ops_playbook_run(
run_code,
requested_by=str(payload.get("requested_by") or "api").strip() or "api",
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/playbook-runs/{run_code}/cancel", response_model=ApiResponse)
def ops_playbook_run_cancel(run_code: str, payload: dict | None = None) -> ApiResponse:
payload = payload or {}
ok, message, data = cancel_ops_playbook_run(
run_code,
cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api",
reason=str(payload.get("reason") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/playbooks/preview", response_model=ApiResponse)
def ops_playbook_preview(payload: dict) -> ApiResponse:
ok, message, data = preview_ops_playbook(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/playbooks/execute", response_model=ApiResponse)
def ops_playbook_execute(payload: dict) -> ApiResponse:
ok, message, data = execute_ops_playbook(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/blueprint", response_model=ApiResponse)
def ops_blueprint() -> ApiResponse:
return ApiResponse(data=get_ops_blueprint())
@router.get("/ops/runbook", response_model=ApiResponse)
def ops_runbook() -> ApiResponse:
return ApiResponse(data=get_ops_runbook())
@router.post("/ops/runbook/sequences/{sequence_key}/resolve", response_model=ApiResponse)
def ops_runbook_sequence_resolve(sequence_key: str, payload: dict | None = None) -> ApiResponse:
ok, message, data = resolve_ops_runbook_sequence(sequence_key, payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/runbook/sequences/{sequence_key}/execute", response_model=ApiResponse)
def ops_runbook_sequence_execute(sequence_key: str, payload: dict | None = None) -> ApiResponse:
ok, message, data = execute_ops_runbook_sequence(sequence_key, payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/nodes", response_model=ApiResponse)
def ops_nodes() -> ApiResponse:
return ApiResponse(data=list_managed_nodes_with_agent_state())
@router.get("/ops/nodes/{node_code}/handover", response_model=ApiResponse)
def ops_node_handover(
node_code: str,
control_plane_base_url: str = "",
root_dir: str = "",
) -> ApiResponse:
data = get_managed_node_handover(
node_code,
control_plane_base_url=control_plane_base_url,
root_dir=root_dir,
)
if not data:
return ApiResponse(code=1, message="节点不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/nodes/{node_code}/onboarding", response_model=ApiResponse)
def ops_node_onboarding(
node_code: str,
control_plane_base_url: str = "",
root_dir: str = "",
) -> ApiResponse:
data = get_managed_node_onboarding(
node_code,
control_plane_base_url=control_plane_base_url,
root_dir=root_dir,
)
if not data:
return ApiResponse(code=1, message="节点不存在", data={})
return ApiResponse(data=data)
@router.post("/ops/nodes/{node_code}/onboarding/acceptance/preview", response_model=ApiResponse)
def ops_node_onboarding_acceptance_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = preview_managed_node_onboarding_acceptance(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/preview", response_model=ApiResponse)
def ops_node_onboarding_bootstrap_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = preview_managed_node_onboarding_bootstrap(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/execute", response_model=ApiResponse)
def ops_node_onboarding_bootstrap_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = execute_managed_node_onboarding_bootstrap(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/onboarding/acceptance/execute", response_model=ApiResponse)
def ops_node_onboarding_acceptance_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = execute_managed_node_onboarding_acceptance(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/onboarding/recovery/preview", response_model=ApiResponse)
def ops_node_onboarding_recovery_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = preview_managed_node_onboarding_recovery(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/onboarding/recovery/execute", response_model=ApiResponse)
def ops_node_onboarding_recovery_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = execute_managed_node_onboarding_recovery(
node_code=node_code,
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/nodes/{node_code}/scene-log", response_model=ApiResponse)
def ops_node_scene_log(
node_code: str,
limit: int = 80,
mode: str = "",
) -> ApiResponse:
data = get_ops_node_scene_log(node_code, limit=limit, mode=mode)
if not data:
return ApiResponse(code=1, message="节点不存在", data={})
return ApiResponse(data=data)
@router.post("/ops/nodes/{node_code}/handover/bootstrap-plan", response_model=ApiResponse)
def ops_node_handover_bootstrap_plan(node_code: str, payload: dict | None = None) -> ApiResponse:
normalized_payload = payload or {}
ok, message, data = build_managed_node_handover_bootstrap_plan(
node_code=node_code,
issued_by=str(normalized_payload.get("issued_by") or "api").strip() or "api",
expires_in_hours=int(normalized_payload.get("expires_in_hours") or 72),
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
metadata=normalized_payload.get("metadata") or {},
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/nodes/{node_code}/delivery-queue", response_model=ApiResponse)
def ops_node_delivery_queue(node_code: str) -> ApiResponse:
data = get_managed_node_delivery_queue(node_code)
if not data:
return ApiResponse(code=1, message="节点不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/nodes/{node_code}/delivery-queue/records", response_model=ApiResponse)
def ops_node_delivery_queue_records(
node_code: str,
state: str = "",
limit: int = 50,
) -> ApiResponse:
data = list_managed_node_delivery_queue_records(
node_code,
state=state,
limit=limit,
)
if not data:
return ApiResponse(code=1, message="节点不存在", data={})
return ApiResponse(data=data)
@router.post("/ops/nodes/{node_code}/delivery-queue/flush", response_model=ApiResponse)
def ops_node_delivery_queue_flush(node_code: str, payload: dict | None = None) -> ApiResponse:
ok, message, data = request_managed_node_delivery_queue_action(
node_code,
"delivery.queue.flush",
payload=payload or {},
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/delivery-queue/replay", response_model=ApiResponse)
def ops_node_delivery_queue_replay(node_code: str, payload: dict | None = None) -> ApiResponse:
ok, message, data = request_managed_node_delivery_queue_action(
node_code,
"delivery.queue.replay",
payload=payload or {},
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", response_model=ApiResponse)
def ops_node_delivery_queue_record_replay(
node_code: str,
record_id: str,
payload: dict | None = None,
) -> ApiResponse:
ok, message, data = request_managed_node_delivery_queue_action(
node_code,
"delivery.queue.replay",
payload=payload or {},
record_id=record_id,
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", response_model=ApiResponse)
def ops_node_delivery_queue_record_discard(
node_code: str,
record_id: str,
payload: dict | None = None,
) -> ApiResponse:
ok, message, data = request_managed_node_delivery_queue_action(
node_code,
"delivery.queue.discard",
payload=payload or {},
record_id=record_id,
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes", response_model=ApiResponse)
def ops_nodes_upsert(payload: dict) -> ApiResponse:
ok, message, data = upsert_managed_node(payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/nodes/sync-from-cluster", response_model=ApiResponse)
def ops_nodes_sync_from_cluster(dry_run: bool = False) -> ApiResponse:
return ApiResponse(data=sync_managed_nodes_from_cluster(dry_run=dry_run))
@router.get("/ops/jobs", response_model=ApiResponse)
def ops_jobs(limit: int = 20, compact: bool = True, rollout_id: int = 0) -> ApiResponse:
normalized_rollout_id = int(rollout_id or 0)
jobs = list_ops_jobs(
limit=limit,
compact=compact,
rollout_id=normalized_rollout_id if normalized_rollout_id > 0 else None,
)
page_status_counts: dict[str, int] = {}
for item in jobs:
item_status = str(item.get("status") or "").strip() or "unknown"
page_status_counts[item_status] = int(page_status_counts.get(item_status, 0) or 0) + 1
return ApiResponse(
data={
"jobs": jobs,
"filters": {
"limit": int(limit or 20),
"compact": bool(compact),
"rollout_id": normalized_rollout_id,
},
"page_summary": {
"jobs_total": len(jobs),
"status_counts": page_status_counts,
},
}
)
@router.get("/ops/jobs/{job_id}", response_model=ApiResponse)
def ops_job_detail(job_id: int) -> ApiResponse:
data = get_ops_job(job_id)
if not data:
return ApiResponse(code=1, message="任务不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/jobs/{job_id}/events", response_model=ApiResponse)
def ops_job_events(job_id: int, limit: int = 50) -> ApiResponse:
events = list_ops_job_events(job_id, limit=limit)
return ApiResponse(
data={
"events": events,
"summary": summarize_ops_job_events(events, job_id=job_id, limit=limit),
}
)
@router.post("/ops/driver-actions/execute", response_model=ApiResponse)
def ops_driver_action_execute(payload: dict) -> ApiResponse:
ok, message, data = execute_driver_action(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/driver-actions/preview", response_model=ApiResponse)
def ops_driver_action_preview(payload: dict) -> ApiResponse:
ok, message, data = preview_driver_action(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/driver-actions/resolve", response_model=ApiResponse)
def ops_driver_action_resolve(payload: dict | None = None) -> ApiResponse:
ok, message, data = resolve_driver_action_request(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/driver-actions/execute-resolved", response_model=ApiResponse)
def ops_driver_action_execute_resolved(payload: dict | None = None) -> ApiResponse:
ok, message, data = execute_resolved_driver_action(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/codex-actions/resolve", response_model=ApiResponse)
def ops_codex_action_resolve(payload: dict | None = None) -> ApiResponse:
ok, message, data = resolve_codex_action(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/codex-actions/execute", response_model=ApiResponse)
def ops_codex_action_execute(payload: dict | None = None) -> ApiResponse:
ok, message, data = execute_codex_action(payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/jobs", response_model=ApiResponse)
def ops_jobs_create(payload: dict) -> ApiResponse:
ok, message, data = create_ops_job(payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/jobs/batch", response_model=ApiResponse)
def ops_jobs_create_batch(payload: dict) -> ApiResponse:
ok, message, data = create_ops_job_batch(payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/policy/preview", response_model=ApiResponse)
def ops_policy_preview(payload: dict) -> ApiResponse:
return ApiResponse(data=preview_ops_job_policy(payload))
@router.post("/ops/jobs/{job_id}/approve", response_model=ApiResponse)
def ops_job_approve(job_id: int, payload: dict | None = None) -> ApiResponse:
payload = payload or {}
ok, message, data = approve_ops_job(job_id, approved_by=str(payload.get("approved_by") or "api").strip() or "api")
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/jobs/{job_id}/cancel", response_model=ApiResponse)
def ops_job_cancel(job_id: int, payload: dict | None = None) -> ApiResponse:
payload = payload or {}
ok, message, data = cancel_ops_job(
job_id,
cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api",
reason=str(payload.get("reason") or "").strip(),
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/jobs/{job_id}/dispatch", response_model=ApiResponse)
def ops_job_dispatch(job_id: int) -> ApiResponse:
ok, message, data = dispatch_ops_job(job_id)
return ApiResponse(code=0 if ok else 1, message=message, data=data)

View File

@@ -0,0 +1,90 @@
from typing import Optional
from fastapi import APIRouter, Header
from app.schemas.common import ApiResponse
from app.services.ops_agent_service import (
agent_append_job_event,
agent_complete_job,
agent_heartbeat,
agent_mark_job_started,
agent_pull_jobs,
agent_register,
build_node_agent_bootstrap_plan,
issue_node_agent_token,
)
router = APIRouter(tags=["ops-agent"])
def _resolve_agent_token(x_domaincheck_agent_token: Optional[str]) -> str:
return str(x_domaincheck_agent_token or "").strip()
def _build_agent_response(ok: bool, message: str, data: object) -> ApiResponse:
detail_code = None
if isinstance(data, dict):
detail_code = str(data.get("detail_code") or "").strip() or None
return ApiResponse(code=0 if ok else 1, message=message, data=data, detail_code=detail_code)
@router.post("/ops/agent/tokens", response_model=ApiResponse)
def ops_agent_issue_token(payload: dict) -> ApiResponse:
ok, message, data = issue_node_agent_token(
node_code=str(payload.get("node_code") or "").strip(),
issued_by=str(payload.get("issued_by") or "api").strip() or "api",
expires_in_hours=int(payload.get("expires_in_hours") or 72),
metadata=payload.get("metadata") or {},
)
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/bootstrap-plan", response_model=ApiResponse)
def ops_agent_bootstrap_plan(payload: dict) -> ApiResponse:
ok, message, data = build_node_agent_bootstrap_plan(
node_code=str(payload.get("node_code") or "").strip(),
node_region=str(payload.get("node_region") or "mainland").strip() or "mainland",
node_role=str(payload.get("node_role") or "worker").strip() or "worker",
issued_by=str(payload.get("issued_by") or "api").strip() or "api",
expires_in_hours=int(payload.get("expires_in_hours") or 72),
control_plane_base_url=str(payload.get("control_plane_base_url") or "").strip(),
root_dir=str(payload.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck",
metadata=payload.get("metadata") or {},
)
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/register", response_model=ApiResponse)
def ops_agent_register(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_register(payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/heartbeat", response_model=ApiResponse)
def ops_agent_heartbeat(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_heartbeat(payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/pull", response_model=ApiResponse)
def ops_agent_pull(payload: dict, limit: int = 1, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_pull_jobs(payload, token=_resolve_agent_token(x_domaincheck_agent_token), limit=limit)
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/jobs/{job_id}/start", response_model=ApiResponse)
def ops_agent_job_start(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_mark_job_started(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/jobs/{job_id}/complete", response_model=ApiResponse)
def ops_agent_job_complete(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_complete_job(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/jobs/{job_id}/events", response_model=ApiResponse)
def ops_agent_job_event(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_append_job_event(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)

View File

@@ -0,0 +1,175 @@
from __future__ import annotations
from fastapi import APIRouter, Request
from fastapi.responses import FileResponse
from app.schemas.common import ApiResponse
from app.services.ops_release_service import (
advance_release_rollout,
activate_release,
create_release,
create_release_and_smart_rollout_from_latest_package,
create_release_from_latest_package,
create_release_rollout,
create_smart_release_rollout,
get_latest_release_package_metadata,
get_release_launchpad,
get_latest_release,
get_release_package_archive_path,
get_release_package_sha256_path,
get_release,
get_release_rollout,
list_release_rollout_jobs,
list_release_rollouts,
list_releases,
preview_release_from_latest_package,
preview_smart_release_rollout,
preview_smart_release_rollout_from_latest_package,
)
router = APIRouter(tags=["ops-release"])
@router.get("/ops/releases", response_model=ApiResponse)
def ops_releases(limit: int = 20, channel: str | None = None) -> ApiResponse:
return ApiResponse(data={"releases": list_releases(limit=limit, channel=channel)})
@router.get("/ops/releases/latest", response_model=ApiResponse)
def ops_releases_latest(channel: str = "stable") -> ApiResponse:
return ApiResponse(data=get_latest_release(channel=channel))
@router.get("/ops/releases/package-metadata/latest", response_model=ApiResponse)
def ops_releases_package_metadata_latest() -> ApiResponse:
return ApiResponse(data=get_latest_release_package_metadata())
@router.get("/ops/releases/launchpad", response_model=ApiResponse)
def ops_releases_launchpad(request: Request, channel: str = "stable") -> ApiResponse:
derived_base_url = str(request.base_url).rstrip("/")
return ApiResponse(data=get_release_launchpad(control_plane_base_url=derived_base_url, channel=channel))
@router.get("/ops/releases/packages/{package_name}/download")
def ops_release_package_download(package_name: str):
try:
archive_path = get_release_package_archive_path(package_name)
except ValueError:
return ApiResponse(code=1, message="发布包名称不合法", data={})
if not archive_path:
return ApiResponse(code=1, message="发布包不存在", data={})
media_type = "application/octet-stream"
archive_name = archive_path.name.lower()
if archive_name.endswith(".tar.gz"):
media_type = "application/gzip"
elif archive_name.endswith(".zip"):
media_type = "application/zip"
return FileResponse(path=archive_path, filename=archive_path.name, media_type=media_type)
@router.get("/ops/releases/packages/{package_name}/sha256")
def ops_release_package_sha256(package_name: str):
try:
sha256_path = get_release_package_sha256_path(package_name)
except ValueError:
return ApiResponse(code=1, message="发布包名称不合法", data={})
if not sha256_path:
return ApiResponse(code=1, message="发布包校验文件不存在", data={})
return FileResponse(path=sha256_path, filename=sha256_path.name, media_type="text/plain; charset=utf-8")
@router.get("/ops/releases/{release_id}", response_model=ApiResponse)
def ops_releases_detail(release_id: int) -> ApiResponse:
data = get_release(release_id)
if not data:
return ApiResponse(code=1, message="Release 不存在", data={})
return ApiResponse(data=data)
@router.post("/ops/releases", response_model=ApiResponse)
def ops_releases_create(payload: dict) -> ApiResponse:
ok, message, data = create_release(payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/from-package/latest", response_model=ApiResponse)
def ops_releases_create_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse:
derived_base_url = str(request.base_url).rstrip("/")
ok, message, data = create_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/from-package/latest/preview", response_model=ApiResponse)
def ops_releases_preview_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse:
derived_base_url = str(request.base_url).rstrip("/")
ok, message, data = preview_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/from-package/latest/smart-rollout", response_model=ApiResponse)
def ops_releases_create_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse:
derived_base_url = str(request.base_url).rstrip("/")
ok, message, data = create_release_and_smart_rollout_from_latest_package(
payload or {},
control_plane_base_url=derived_base_url,
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/from-package/latest/smart-rollout-preview", response_model=ApiResponse)
def ops_releases_preview_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse:
derived_base_url = str(request.base_url).rstrip("/")
ok, message, data = preview_smart_release_rollout_from_latest_package(
payload or {},
control_plane_base_url=derived_base_url,
)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/{release_id}/activate", response_model=ApiResponse)
def ops_releases_activate(release_id: int) -> ApiResponse:
ok, message, data = activate_release(release_id)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/{release_id}/smart-rollout-preview", response_model=ApiResponse)
def ops_releases_smart_rollout_preview(release_id: int, payload: dict | None = None) -> ApiResponse:
ok, message, data = preview_smart_release_rollout(release_id, payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/{release_id}/smart-rollout", response_model=ApiResponse)
def ops_releases_smart_rollout(release_id: int, payload: dict | None = None) -> ApiResponse:
ok, message, data = create_smart_release_rollout(release_id, payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.get("/ops/releases/{release_id}/rollouts", response_model=ApiResponse)
def ops_release_rollouts(release_id: int, limit: int = 20) -> ApiResponse:
return ApiResponse(data={"rollouts": list_release_rollouts(release_id=release_id, limit=limit)})
@router.get("/ops/rollouts/{rollout_id}", response_model=ApiResponse)
def ops_rollout_detail(rollout_id: int) -> ApiResponse:
data = get_release_rollout(rollout_id)
if not data:
return ApiResponse(code=1, message="Rollout 不存在", data={})
return ApiResponse(data=data)
@router.get("/ops/rollouts/{rollout_id}/jobs", response_model=ApiResponse)
def ops_rollout_jobs(rollout_id: int, limit: int = 200) -> ApiResponse:
return ApiResponse(data={"jobs": list_release_rollout_jobs(rollout_id, limit=limit)})
@router.post("/ops/rollouts/{rollout_id}/advance", response_model=ApiResponse)
def ops_rollout_advance(rollout_id: int, payload: dict | None = None) -> ApiResponse:
ok, message, data = advance_release_rollout(rollout_id, payload or {})
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@router.post("/ops/releases/{release_id}/rollouts", response_model=ApiResponse)
def ops_release_rollouts_create(release_id: int, payload: dict) -> ApiResponse:
ok, message, data = create_release_rollout(release_id, payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)

View File

@@ -3,6 +3,7 @@ from typing import Optional
from fastapi import APIRouter, Header
from app.schemas.common import ApiResponse
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.debug_event_service import get_debug_diagnosis, get_debug_event_overview, get_debug_handoff_report, ingest_debug_event, list_debug_events
from app.services.runtime_control_service import runtime_action
@@ -37,6 +38,11 @@ def runtime_cluster() -> ApiResponse:
return ApiResponse(data=get_cluster_snapshot())
@router.get("/runtime/build-info", response_model=ApiResponse)
def runtime_build_info() -> ApiResponse:
return ApiResponse(data=get_runtime_build_info())
@router.get("/runtime/sync-summary", response_model=ApiResponse)
def runtime_sync_summary() -> ApiResponse:
return ApiResponse(data=get_sync_summary())

View File

@@ -42,6 +42,13 @@ class Settings(BaseSettings):
sync_shared_token: str = ""
sync_batch_size: int = 200
sync_poll_interval_seconds: int = 30
build_manifest_path: str = ""
build_commit_sha: str = ""
build_commit_ref: str = ""
build_generated_at: str = ""
build_package_name: str = ""
build_checksum: str = ""
build_source_label: str = ""
@property
def cors_origins_list(self) -> list[str]:

View File

@@ -3,9 +3,13 @@ import threading
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, sensitive_words
from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, ops, ops_agent, ops_release, sensitive_words
from app.core.config import settings as app_settings
from app.services.build_info_service import get_runtime_build_info, remember_registered_route_paths
from app.services.cluster_runtime_service import ensure_runtime_schema, register_local_control_heartbeat
from app.services.ops_agent_service import ensure_ops_agent_schema
from app.services.ops_job_service import ensure_ops_schema
from app.services.ops_release_service import ensure_ops_release_schema
_heartbeat_stop_event = threading.Event()
@@ -36,6 +40,10 @@ app.add_middleware(
@app.on_event("startup")
def on_startup() -> None:
ensure_runtime_schema()
ensure_ops_schema()
ensure_ops_agent_schema()
ensure_ops_release_schema()
remember_registered_route_paths(route.path for route in app.routes)
register_local_control_heartbeat()
_heartbeat_stop_event.clear()
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
@@ -56,6 +64,7 @@ def health() -> dict:
"worker_mode": app_settings.worker_mode,
"api_host": app_settings.api_host,
"api_port": app_settings.api_port,
"build": get_runtime_build_info(route_paths=(route.path for route in app.routes)),
}
@@ -69,4 +78,8 @@ app.include_router(exports.router, prefix=app_settings.api_prefix)
app.include_router(logs.router, prefix=app_settings.api_prefix)
app.include_router(runtime.router, prefix=app_settings.api_prefix)
app.include_router(juming.router, prefix=app_settings.api_prefix)
app.include_router(ops.router, prefix=app_settings.api_prefix)
app.include_router(ops_agent.router, prefix=app_settings.api_prefix)
app.include_router(ops_release.router, prefix=app_settings.api_prefix)
app.include_router(sensitive_words.router, prefix=app_settings.api_prefix)
remember_registered_route_paths(route.path for route in app.routes)

View File

@@ -0,0 +1,991 @@
from __future__ import annotations
import json
import os
import socket
import subprocess
import time
import urllib.error
import urllib.request
from datetime import datetime
from uuid import uuid4
from app.services.ops_action_executor_core import (
build_service_name_map,
execute_structured_action,
supports_structured_action,
)
from app.services.ops_release_executor_core import (
execute_release_action,
normalize_release_health_check_services as _release_normalize_health_check_services,
normalize_text_list as _release_normalize_text_list,
)
CONTROL_PLANE_BASE_URL = str(os.getenv("OPS_CONTROL_PLANE_BASE_URL", "")).strip().rstrip("/")
AGENT_TOKEN = str(os.getenv("OPS_AGENT_TOKEN", "")).strip()
NODE_CODE = str(os.getenv("NODE_CODE", "")).strip()
NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland"
NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker"
AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5))
WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker"
API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api"
SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent"
NODE_AGENT_SERVICE_NAME = str(os.getenv("NODE_AGENT_SERVICE_NAME", "domaincheck-node-agent")).strip() or "domaincheck-node-agent"
AGENT_VERSION = "0.1.0"
_APP_DIR = os.path.dirname(os.path.abspath(__file__))
_PROJECT_DIR = os.path.dirname(_APP_DIR)
_DEFAULT_QUEUE_DIR = os.path.join(_PROJECT_DIR, "runtime", "node-agent-queue", NODE_CODE or "unbound")
AGENT_QUEUE_DIR = str(os.getenv("OPS_AGENT_QUEUE_DIR", _DEFAULT_QUEUE_DIR)).strip() or _DEFAULT_QUEUE_DIR
AGENT_QUEUE_FLUSH_LIMIT = max(1, int(os.getenv("OPS_AGENT_QUEUE_FLUSH_LIMIT", "20") or 20))
_PERMANENT_DELIVERY_DETAIL_CODES = {
"agent_node_code_required",
"ops_job_not_owned_by_agent",
"ops_job_invalid_status",
}
_LAST_QUEUE_FLUSH_SUMMARY = {
"scanned": 0,
"delivered": 0,
"deferred": 0,
"dead_letter": 0,
"last_flush_at": "",
}
def _normalize_text_list(raw_value: object) -> list[str]:
"""Keep node-agent payload normalization aligned with release executor helpers."""
return _release_normalize_text_list(raw_value)
def _normalize_release_health_check_services(
payload: dict | None,
restart_services: list[str] | None,
) -> list[str]:
"""Backward-compatible wrapper used by node-agent tests and payload shaping."""
return _release_normalize_health_check_services(
dict(payload or {}),
list(restart_services or []),
default_api_service_name=API_SERVICE_NAME,
)
def _log(message: str) -> None:
print(f"{datetime.now().isoformat(sep=' ', timespec='seconds')} [node-agent] {message}", flush=True)
def _json_env(name: str, fallback: object) -> object:
raw = str(os.getenv(name, "")).strip()
if not raw:
return fallback
try:
return json.loads(raw)
except Exception:
return fallback
AGENT_CAPABILITIES = _json_env(
"OPS_AGENT_CAPABILITIES",
[
"service.start",
"service.stop",
"service.restart",
"service.status",
"runtime.start_worker",
"runtime.stop_worker",
"runtime.restart_api",
"runtime.start_sync_agent",
"runtime.stop_sync_agent",
"health.snapshot",
"logs.collect",
"diagnostics.collect",
"delivery.queue.flush",
"delivery.queue.replay",
"delivery.queue.discard",
"deploy.release",
],
)
AGENT_LABELS = _json_env("OPS_AGENT_LABELS", {})
class AgentResponseError(RuntimeError):
def __init__(self, message: str, *, detail_code: str = "", payload: dict | None = None):
super().__init__(message)
self.detail_code = str(detail_code or "").strip()
self.payload = dict(payload or {})
def _now_text() -> str:
return datetime.now().isoformat(sep=" ", timespec="seconds")
def _queue_pending_dir() -> str:
return os.path.join(AGENT_QUEUE_DIR, "pending")
def _queue_dead_letter_dir() -> str:
return os.path.join(AGENT_QUEUE_DIR, "dead-letter")
def _queue_discarded_dir() -> str:
return os.path.join(AGENT_QUEUE_DIR, "discarded")
def _ensure_queue_dirs() -> None:
for path in (AGENT_QUEUE_DIR, _queue_pending_dir(), _queue_dead_letter_dir(), _queue_discarded_dir()):
os.makedirs(path, exist_ok=True)
def _write_json_file(path: str, payload: dict) -> None:
temp_path = f"{path}.tmp"
with open(temp_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle, ensure_ascii=False, indent=2)
os.replace(temp_path, path)
def _read_json_file(path: str) -> dict:
with open(path, "r", encoding="utf-8") as handle:
return json.load(handle)
def _new_request_id(prefix: str, job_id: int) -> str:
normalized_prefix = str(prefix or "request").strip() or "request"
return f"{normalized_prefix}-{int(job_id or 0)}-{int(time.time() * 1000)}-{uuid4().hex[:10]}"
def _queue_file_names(directory: str) -> list[str]:
_ensure_queue_dirs()
try:
return sorted(file_name for file_name in os.listdir(directory) if file_name.endswith(".json"))
except FileNotFoundError:
return []
def _queue_head_record(directory: str) -> dict:
file_names = _queue_file_names(directory)
if not file_names:
return {}
file_path = os.path.join(directory, file_names[0])
try:
return _read_json_file(file_path)
except Exception:
return {}
def _queue_record_id(record: dict, *, file_name: str = "") -> str:
request_id = str((record or {}).get("request_id") or "").strip()
if request_id:
return request_id
normalized_file_name = str(file_name or "").strip()
if normalized_file_name.endswith(".json"):
return normalized_file_name[:-5]
return normalized_file_name
def _queue_entries(state: str) -> list[dict]:
normalized_state = str(state or "").strip()
if normalized_state == "pending":
directory = _queue_pending_dir()
elif normalized_state == "dead_letter":
directory = _queue_dead_letter_dir()
else:
return []
entries: list[dict] = []
for file_name in _queue_file_names(directory):
file_path = os.path.join(directory, file_name)
try:
record = _read_json_file(file_path)
except Exception:
continue
entries.append(
{
"state": normalized_state,
"file_name": file_name,
"file_path": file_path,
"record_id": _queue_record_id(record, file_name=file_name),
"record": dict(record or {}),
}
)
return entries
def _matches_delivery_selector(entry: dict, selector: dict, *, allowed_states: set[str] | None = None) -> bool:
normalized_selector = dict(selector or {})
record = dict(entry.get("record") or {})
state = str(entry.get("state") or "").strip()
if allowed_states and state not in allowed_states:
return False
selected_state = str(normalized_selector.get("state") or "").strip()
if selected_state and state != selected_state:
return False
selected_record_id = str(
normalized_selector.get("record_id")
or normalized_selector.get("request_id")
or ""
).strip()
if selected_record_id and str(entry.get("record_id") or "").strip() != selected_record_id:
return False
selected_request_kind = str(normalized_selector.get("request_kind") or "").strip()
if selected_request_kind and str(record.get("kind") or "").strip() != selected_request_kind:
return False
selected_detail_code = str(normalized_selector.get("detail_code") or "").strip()
if selected_detail_code and str(record.get("last_detail_code") or "").strip() != selected_detail_code:
return False
return True
def _trim_queue_preview(records: list[dict], *, limit: int = 5) -> list[dict]:
safe_limit = max(1, int(limit or 5))
preview: list[dict] = []
for entry in list(records or [])[:safe_limit]:
record = dict(entry.get("record") or {})
preview.append(
{
"record_id": str(entry.get("record_id") or "").strip(),
"state": str(entry.get("state") or "").strip(),
"request_kind": str(record.get("kind") or "").strip(),
"request_id": str(record.get("request_id") or "").strip(),
"detail_code": str(record.get("last_detail_code") or "").strip(),
"created_at": str(record.get("created_at") or "").strip(),
"updated_at": str(record.get("updated_at") or "").strip(),
}
)
return preview
def _normalize_queue_limit(raw_value: object, *, default: int = 20, minimum: int = 1, maximum: int = 200) -> int:
try:
value = int(raw_value or default)
except Exception:
value = int(default)
return max(minimum, min(maximum, value))
def _replay_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
selector = dict(normalized_payload.get("selector") or {})
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
flush_after_replay = bool(normalized_payload.get("flush_after_replay", True))
reason = str(normalized_payload.get("reason") or "").strip()
matched_entries = [
entry
for entry in _queue_entries("dead_letter")
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
][:limit]
if not matched_entries:
return False, "未找到符合条件的死信记录", {
"selector": selector,
"limit": limit,
"queue": _delivery_queue_snapshot(),
}
replayed_total = 0
for entry in matched_entries:
record = dict(entry.get("record") or {})
record["updated_at"] = _now_text()
record["replay_count"] = int(record.get("replay_count") or 0) + 1
record["last_replay_at"] = _now_text()
if reason:
record["last_replay_reason"] = reason
record.pop("dead_letter_at", None)
record.pop("dead_letter_reason", None)
_store_pending_delivery(record)
try:
os.remove(str(entry.get("file_path") or ""))
except FileNotFoundError:
pass
replayed_total += 1
flush_summary = {}
if flush_after_replay and replayed_total > 0:
flush_summary = _flush_delivery_queue(limit=replayed_total)
result = {
"selector": selector,
"limit": limit,
"replayed_total": replayed_total,
"flush_after_replay": flush_after_replay,
"flush_summary": flush_summary,
"matched_records": _trim_queue_preview(matched_entries),
"queue": _delivery_queue_snapshot(),
}
return True, f"已重放 {replayed_total} 条死信记录", result
def _store_discarded_delivery(record: dict, *, discarded_by: str = "", reason: str = "") -> str:
_ensure_queue_dirs()
discarded_record = dict(record or {})
discarded_record["discarded_at"] = _now_text()
discarded_record["discarded_by"] = str(discarded_by or "").strip()
discarded_record["discard_reason"] = str(reason or "").strip()
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
file_path = os.path.join(_queue_discarded_dir(), file_name)
_write_json_file(file_path, discarded_record)
return file_path
def _discard_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
selector = dict(normalized_payload.get("selector") or {})
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
discarded_by = str(normalized_payload.get("discarded_by") or "").strip()
reason = str(normalized_payload.get("reason") or "").strip()
matched_entries = [
entry
for entry in _queue_entries("dead_letter")
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
][:limit]
if not matched_entries:
return False, "未找到符合条件的死信记录", {
"selector": selector,
"limit": limit,
"queue": _delivery_queue_snapshot(),
}
discarded_total = 0
for entry in matched_entries:
_store_discarded_delivery(
dict(entry.get("record") or {}),
discarded_by=discarded_by,
reason=reason,
)
try:
os.remove(str(entry.get("file_path") or ""))
except FileNotFoundError:
pass
discarded_total += 1
result = {
"selector": selector,
"limit": limit,
"discarded_total": discarded_total,
"discarded_by": discarded_by,
"reason": reason,
"matched_records": _trim_queue_preview(matched_entries),
"queue": _delivery_queue_snapshot(),
}
return True, f"已丢弃 {discarded_total} 条死信记录", result
def _delivery_queue_snapshot() -> dict:
pending_dir = _queue_pending_dir()
dead_letter_dir = _queue_dead_letter_dir()
pending_files = _queue_file_names(pending_dir)
dead_letter_files = _queue_file_names(dead_letter_dir)
pending_head = _queue_head_record(pending_dir)
dead_letter_head = _queue_head_record(dead_letter_dir)
pending_count = len(pending_files)
dead_letter_count = len(dead_letter_files)
state = "healthy"
label = "正常"
reason = "当前没有待重试回执,也没有死信记录。"
if dead_letter_count > 0:
state = "dead_letter"
label = f"死信 {dead_letter_count}"
reason = "存在语义失败的回执/事件,自动重试已停止,建议人工查看。"
elif pending_count > 0:
state = "retrying"
label = f"待重试 {pending_count}"
reason = "存在待重试的回执/事件Node Agent 会在后续 heartbeat/poll 周期继续回放。"
elif not pending_head and not dead_letter_head:
label = "正常"
return {
"state": state,
"label": label,
"reason": reason,
"pending_count": pending_count,
"dead_letter_count": dead_letter_count,
"oldest_pending_at": str(pending_head.get("created_at") or "").strip(),
"oldest_pending_request_id": str(pending_head.get("request_id") or "").strip(),
"oldest_pending_kind": str(pending_head.get("kind") or "").strip(),
"oldest_dead_letter_at": str(
dead_letter_head.get("dead_letter_at") or dead_letter_head.get("created_at") or ""
).strip(),
"oldest_dead_letter_request_id": str(dead_letter_head.get("request_id") or "").strip(),
"oldest_dead_letter_kind": str(dead_letter_head.get("kind") or "").strip(),
"last_flush_at": str(_LAST_QUEUE_FLUSH_SUMMARY.get("last_flush_at") or "").strip(),
"last_flush_delivered": int(_LAST_QUEUE_FLUSH_SUMMARY.get("delivered") or 0),
"last_flush_deferred": int(_LAST_QUEUE_FLUSH_SUMMARY.get("deferred") or 0),
"last_flush_dead_letter": int(_LAST_QUEUE_FLUSH_SUMMARY.get("dead_letter") or 0),
}
def _response_detail_code(response: dict) -> str:
if not isinstance(response, dict):
return ""
top_level = str(response.get("detail_code") or "").strip()
if top_level:
return top_level
data = response.get("data")
if isinstance(data, dict):
return str(data.get("detail_code") or "").strip()
return ""
def _ensure_ok_response(response: dict, fallback_message: str) -> dict:
raw_code = response.get("code", 1)
try:
normalized_code = int(raw_code if raw_code not in (None, "") else 1)
except Exception:
normalized_code = 1
if normalized_code == 0:
return response
raise AgentResponseError(
str(response.get("message") or fallback_message),
detail_code=_response_detail_code(response),
payload=(response.get("data") if isinstance(response.get("data"), dict) else {}),
)
def _build_delivery_record(kind: str, path: str, payload: dict, request_id: str) -> dict:
return {
"kind": str(kind or "").strip() or "delivery",
"path": str(path or "").strip(),
"payload": dict(payload or {}),
"request_id": str(request_id or "").strip(),
"attempt_count": 0,
"created_at": _now_text(),
"updated_at": _now_text(),
"last_error": "",
"last_detail_code": "",
}
def _store_pending_delivery(record: dict) -> str:
_ensure_queue_dirs()
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
file_path = os.path.join(_queue_pending_dir(), file_name)
_write_json_file(file_path, record)
return file_path
def _store_dead_letter_delivery(record: dict, *, reason: str, detail_code: str = "") -> str:
_ensure_queue_dirs()
dead_record = dict(record or {})
dead_record["dead_letter_reason"] = str(reason or "").strip()
dead_record["dead_letter_at"] = _now_text()
dead_record["last_error"] = str(reason or "").strip()
dead_record["last_detail_code"] = str(detail_code or "").strip()
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
file_path = os.path.join(_queue_dead_letter_dir(), file_name)
_write_json_file(file_path, dead_record)
return file_path
def _dispatch_delivery_record(record: dict) -> dict:
kind = str(record.get("kind") or "delivery").strip() or "delivery"
path = str(record.get("path") or "").strip()
payload = record.get("payload") or {}
timeout = 15 if kind == "job_event" else 30
response = _post(path, payload, timeout=timeout)
return _ensure_ok_response(response, f"{kind} failed")
def _flush_delivery_queue(limit: int | None = None) -> dict:
global _LAST_QUEUE_FLUSH_SUMMARY
_ensure_queue_dirs()
safe_limit = max(1, int(limit or AGENT_QUEUE_FLUSH_LIMIT or 1))
summary = {
"scanned": 0,
"delivered": 0,
"deferred": 0,
"dead_letter": 0,
}
file_names = sorted(
file_name
for file_name in os.listdir(_queue_pending_dir())
if file_name.endswith(".json")
)[:safe_limit]
for file_name in file_names:
summary["scanned"] += 1
file_path = os.path.join(_queue_pending_dir(), file_name)
try:
record = _read_json_file(file_path)
except Exception as exc:
_store_dead_letter_delivery({"file_name": file_name}, reason=f"invalid queue record: {exc}")
try:
os.remove(file_path)
except FileNotFoundError:
pass
summary["dead_letter"] += 1
continue
try:
_dispatch_delivery_record(record)
try:
os.remove(file_path)
except FileNotFoundError:
pass
summary["delivered"] += 1
except AgentResponseError as exc:
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
record["updated_at"] = _now_text()
record["last_error"] = str(exc)
record["last_detail_code"] = exc.detail_code
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
_store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
try:
os.remove(file_path)
except FileNotFoundError:
pass
summary["dead_letter"] += 1
_log(
f"delivery moved to dead-letter: kind={record.get('kind')} request_id={record.get('request_id')} "
f"detail_code={exc.detail_code or '-'} message={exc}"
)
else:
_write_json_file(file_path, record)
summary["deferred"] += 1
except Exception as exc:
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
record["updated_at"] = _now_text()
record["last_error"] = str(exc)
_write_json_file(file_path, record)
summary["deferred"] += 1
summary["last_flush_at"] = _now_text()
_LAST_QUEUE_FLUSH_SUMMARY = dict(summary)
return summary
def _deliver_or_queue(
*,
kind: str,
path: str,
payload: dict,
request_id: str,
timeout: int = 30,
) -> dict:
record = _build_delivery_record(kind, path, payload, request_id)
try:
response = _post(path, payload, timeout=timeout)
_ensure_ok_response(response, f"{kind} failed")
return {"state": "delivered", "request_id": request_id, "detail_code": "", "response": response}
except AgentResponseError as exc:
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
file_path = _store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
_log(
f"{kind} dead-lettered: request_id={request_id} detail_code={exc.detail_code or '-'} "
f"message={exc} file={file_path}"
)
return {"state": "dead_letter", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
file_path = _store_pending_delivery({**record, "last_error": str(exc), "last_detail_code": exc.detail_code})
_log(
f"{kind} queued for retry: request_id={request_id} detail_code={exc.detail_code or '-'} "
f"message={exc} file={file_path}"
)
return {"state": "queued", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
except Exception as exc:
file_path = _store_pending_delivery({**record, "last_error": str(exc)})
_log(f"{kind} queued for retry: request_id={request_id} message={exc} file={file_path}")
return {"state": "queued", "request_id": request_id, "detail_code": "", "file_path": file_path}
def _headers() -> dict[str, str]:
return {
"Content-Type": "application/json",
"X-Domaincheck-Agent-Token": AGENT_TOKEN,
}
def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict:
if not CONTROL_PLANE_BASE_URL:
raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置")
if not AGENT_TOKEN:
raise RuntimeError("OPS_AGENT_TOKEN 未配置")
url = f"{CONTROL_PLANE_BASE_URL}{path}"
data = json.dumps(payload or {}, ensure_ascii=False).encode("utf-8")
request = urllib.request.Request(url=url, data=data, headers=_headers(), method=method.upper())
with urllib.request.urlopen(request, timeout=timeout) as response:
body = response.read().decode("utf-8", errors="ignore")
return json.loads(body or "{}")
def _post(path: str, payload: dict, timeout: int = 30) -> dict:
return _request("POST", path, payload, timeout=timeout)
def _hostname() -> str:
try:
return socket.gethostname()
except Exception:
return ""
def _ip() -> str:
try:
return socket.gethostbyname(socket.gethostname())
except Exception:
return ""
def _base_payload() -> dict:
return {
"node_code": NODE_CODE,
"region": NODE_REGION,
"role": NODE_ROLE,
"title": NODE_CODE,
"hostname": _hostname(),
"ip": _ip(),
"agent_version": AGENT_VERSION,
"capabilities": AGENT_CAPABILITIES,
"labels": AGENT_LABELS,
"metadata": {
"service_names": {
"api": API_SERVICE_NAME,
"worker": WORKER_SERVICE_NAME,
"sync_agent": SYNC_AGENT_SERVICE_NAME,
"node_agent": NODE_AGENT_SERVICE_NAME,
},
"delivery_queue": _delivery_queue_snapshot(),
},
}
def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]:
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
return completed.returncode, completed.stdout.strip(), completed.stderr.strip()
def _execute_action(
action: str,
payload: dict,
*,
job_id: int | None = None,
job_context: dict | None = None,
) -> tuple[bool, str, dict]:
normalized_action = str(action or "").strip()
if normalized_action == "delivery.queue.flush":
limit = _normalize_queue_limit((payload or {}).get("limit"), default=20)
summary = _flush_delivery_queue(limit=limit)
return True, "Delivery Queue 已执行冲刷", {
"action": normalized_action,
"limit": limit,
"flush_summary": summary,
"queue": _delivery_queue_snapshot(),
}
if normalized_action == "delivery.queue.replay":
return _replay_dead_letter_records(payload)
if normalized_action == "delivery.queue.discard":
return _discard_dead_letter_records(payload)
if supports_structured_action(normalized_action):
return execute_structured_action(
normalized_action,
payload,
service_names=build_service_name_map(
api_service_name=API_SERVICE_NAME,
worker_service_name=WORKER_SERVICE_NAME,
sync_agent_service_name=SYNC_AGENT_SERVICE_NAME,
node_agent_service_name=NODE_AGENT_SERVICE_NAME,
),
runner=_run,
host_context={
"hostname": _hostname(),
"ip": _ip(),
},
)
if normalized_action == "deploy.release":
return execute_release_action(
dict(payload or {}),
run_command=_run,
default_api_service_name=API_SERVICE_NAME,
event_callback=(
(lambda event_type, message, level="info", payload=None: _job_event(
int(job_id),
event_type=event_type,
message=message,
level=level,
payload={
**dict(payload or {}),
"release_context": dict((job_context or {}).get("release_context") or {}),
"step_key": str((job_context or {}).get("step_key") or "").strip(),
},
summary_text=message,
focus_ref=dict((job_context or {}).get("focus_ref") or {}),
occurred_at=_now_text(),
))
if job_id
else None
),
urlopen_func=urllib.request.urlopen,
user_agent=f"domaincheck-node-agent/{AGENT_VERSION}",
)
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}
def _register() -> None:
response = _post("/api/v1/ops/agent/register", _base_payload())
_ensure_ok_response(response, "agent register failed")
_log(f"registered: {response.get('message')}")
def _heartbeat() -> None:
response = _post("/api/v1/ops/agent/heartbeat", _base_payload())
_ensure_ok_response(response, "agent heartbeat failed")
def _pull_jobs() -> list[dict]:
response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE})
_ensure_ok_response(response, "agent pull failed")
data = response.get("data") or {}
return list(data.get("jobs") or [])
def _normalize_agent_job(job: dict) -> dict:
normalized_job = dict(job or {})
focus_ref = normalized_job.get("focus_ref") if isinstance(normalized_job.get("focus_ref"), dict) else {}
release_context = (
normalized_job.get("release_context") if isinstance(normalized_job.get("release_context"), dict) else {}
)
step_ref = normalized_job.get("step_ref") if isinstance(normalized_job.get("step_ref"), dict) else {}
job_id = int(normalized_job.get("job_id") or normalized_job.get("id") or 0)
step_key = str(
normalized_job.get("step_key")
or step_ref.get("step_key")
or "dispatch"
).strip() or "dispatch"
step_title = str(
normalized_job.get("step_title")
or step_ref.get("step_title")
or normalized_job.get("summary")
or normalized_job.get("action")
or step_key
).strip() or step_key
if not focus_ref:
focus_ref = {
"kind": "ops_job",
"job_id": job_id,
"job_code": str(normalized_job.get("job_code") or "").strip(),
"action": str(normalized_job.get("action") or "").strip(),
"target_node_code": str(normalized_job.get("target_node_code") or "").strip(),
}
return {
**normalized_job,
"job_id": job_id,
"job_code": str(normalized_job.get("job_code") or "").strip(),
"job_type": str(normalized_job.get("job_type") or "ops_action").strip() or "ops_action",
"action": str(normalized_job.get("action") or "").strip(),
"payload": dict(normalized_job.get("payload") or {}),
"policy": dict(normalized_job.get("policy") or {}),
"focus_ref": focus_ref,
"release_context": dict(release_context or {}),
"step_key": step_key,
"step_title": step_title,
"step_ref": {
"step_id": int(step_ref.get("step_id") or 0),
"step_key": step_key,
"step_title": step_title,
},
}
def _job_start(job_id: int, *, job: dict | None = None) -> None:
normalized_job = _normalize_agent_job(job or {"job_id": job_id})
response = _post(
f"/api/v1/ops/agent/jobs/{job_id}/start",
{
"node_code": NODE_CODE,
"job_code": normalized_job.get("job_code") or "",
"action": normalized_job.get("action") or "",
"step_key": normalized_job.get("step_key") or "",
"focus_ref": normalized_job.get("focus_ref") or {},
},
)
_ensure_ok_response(response, "job start failed")
def _job_complete(
job_id: int,
*,
status: str,
stdout: str,
stderr: str,
result: dict,
error_message: str = "",
client_request_id: str | None = None,
duration_ms: int | None = None,
summary_text: str = "",
focus_ref: dict | None = None,
step_ref: dict | None = None,
release_context: dict | None = None,
) -> dict:
request_id = str(client_request_id or "").strip() or _new_request_id("complete", job_id)
payload = {
"node_code": NODE_CODE,
"status": status,
"stdout": stdout,
"stderr": stderr,
"result": result,
"error_message": error_message,
"client_request_id": request_id,
}
if duration_ms is not None:
payload["duration_ms"] = max(0, int(duration_ms or 0))
if str(summary_text or "").strip():
payload["summary_text"] = str(summary_text).strip()
if isinstance(focus_ref, dict) and focus_ref:
payload["focus_ref"] = dict(focus_ref)
if isinstance(step_ref, dict) and step_ref:
payload["step_ref"] = dict(step_ref)
if isinstance(release_context, dict) and release_context:
payload["release_context"] = dict(release_context)
return _deliver_or_queue(
kind="job_complete",
path=f"/api/v1/ops/agent/jobs/{job_id}/complete",
payload=payload,
request_id=request_id,
timeout=30,
)
def _job_event(
job_id: int,
*,
event_type: str,
message: str,
level: str = "info",
payload: dict | None = None,
client_event_id: str | None = None,
summary_text: str = "",
focus_ref: dict | None = None,
occurred_at: str = "",
) -> dict:
request_id = str(client_event_id or "").strip() or _new_request_id("event", job_id)
delivery_payload = {
"node_code": NODE_CODE,
"event_type": event_type,
"message": message,
"level": level,
"payload": payload or {},
"client_event_id": request_id,
}
if str(summary_text or "").strip():
delivery_payload["summary_text"] = str(summary_text).strip()
if isinstance(focus_ref, dict) and focus_ref:
delivery_payload["focus_ref"] = dict(focus_ref)
if str(occurred_at or "").strip():
delivery_payload["occurred_at"] = str(occurred_at).strip()
return _deliver_or_queue(
kind="job_event",
path=f"/api/v1/ops/agent/jobs/{job_id}/events",
payload=delivery_payload,
request_id=request_id,
timeout=15,
)
def _process_job(job: dict) -> None:
normalized_job = _normalize_agent_job(job)
job_id = int(normalized_job.get("job_id") or 0)
action = str(normalized_job.get("action") or "").strip()
payload = dict(normalized_job.get("payload") or {})
if job_id <= 0 or not action:
return
started_at = time.monotonic()
_log(
"job start: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"type={normalized_job.get('job_type') or '-'} "
f"step={normalized_job.get('step_key') or '-'} action={action}"
)
start_delivery_state = "delivered"
start_delivery_error = ""
try:
_job_start(job_id, job=normalized_job)
except Exception as exc:
start_delivery_state = "failed_local"
start_delivery_error = str(exc)
_log(
"job start delivery failed, continue locally: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} action={action} error={exc}"
)
_job_event(
job_id,
event_type="executor_received",
message=f"node agent accepted action {action}",
summary_text=f"已接单 {action}",
focus_ref=dict(normalized_job.get("focus_ref") or {}),
occurred_at=_now_text(),
payload={
"action": action,
"job_code": normalized_job.get("job_code") or "",
"job_type": normalized_job.get("job_type") or "",
"step_key": normalized_job.get("step_key") or "",
"step_title": normalized_job.get("step_title") or "",
"release_context": dict(normalized_job.get("release_context") or {}),
"start_delivery_state": start_delivery_state,
"start_delivery_error": start_delivery_error,
},
)
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
stdout = str(result.get("stdout") or "")
stderr = str(result.get("stderr") or "")
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
summary_text = str(result.get("summary_text") or result.get("summary") or message).strip()
delivery = _job_complete(
job_id,
status="success" if ok else "failed",
stdout=stdout,
stderr=stderr,
result=result,
error_message="" if ok else message,
duration_ms=duration_ms,
summary_text=summary_text,
focus_ref=dict(normalized_job.get("focus_ref") or {}),
step_ref=dict(normalized_job.get("step_ref") or {}),
release_context=dict(normalized_job.get("release_context") or {}),
)
_log(
"job complete: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action} ok={ok} duration_ms={duration_ms} delivery={delivery.get('state')}"
)
def main() -> None:
if not NODE_CODE:
raise RuntimeError("NODE_CODE 未配置")
_log(f"starting node agent: node={NODE_CODE} role={NODE_ROLE} region={NODE_REGION}")
_ensure_queue_dirs()
_register()
last_heartbeat_at = 0.0
while True:
now = time.time()
try:
delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT)
if delivery_summary["delivered"] or delivery_summary["dead_letter"]:
_log(f"delivery queue flush: {delivery_summary}")
if now - last_heartbeat_at >= 15:
_heartbeat()
last_heartbeat_at = now
jobs = _pull_jobs()
if jobs:
for job in jobs:
_process_job(job)
else:
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
except urllib.error.HTTPError as exc:
_log(f"http error: {exc.code}")
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
except urllib.error.URLError as exc:
_log(f"url error: {exc}")
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
except Exception as exc:
_log(f"loop error: {exc}")
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
if __name__ == "__main__":
main()

View File

@@ -1,8 +1,10 @@
from typing import Any, Optional
from pydantic import BaseModel
from typing import Any
class ApiResponse(BaseModel):
code: int = 0
message: str = "ok"
data: Any = None
detail_code: Optional[str] = None

View File

@@ -0,0 +1,241 @@
from __future__ import annotations
import json
import subprocess
from pathlib import Path
from typing import Iterable
from app.core.config import settings
DOMAIN_API_ROOT = Path(__file__).resolve().parents[2]
WORKSPACE_ROOT = DOMAIN_API_ROOT.parent
_REGISTERED_ROUTE_PATHS: set[str] = set()
def remember_registered_route_paths(route_paths: Iterable[str] | None) -> None:
global _REGISTERED_ROUTE_PATHS
normalized = {
str(item).strip()
for item in list(route_paths or [])
if str(item).strip()
}
_REGISTERED_ROUTE_PATHS = normalized
def _read_json(path: Path) -> dict:
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {}
def _discover_manifest_candidate() -> tuple[Path | None, str]:
configured = str(settings.build_manifest_path or "").strip()
if configured:
path = Path(configured)
if path.exists():
return path, "configured_manifest"
root_manifest = WORKSPACE_ROOT / "release_manifest.json"
if root_manifest.exists():
return root_manifest, "release_manifest"
latest_release = WORKSPACE_ROOT / "release" / "latest_release.json"
if latest_release.exists():
return latest_release, "latest_release"
return None, ""
def _build_info_from_env() -> dict | None:
if not any(
[
str(settings.build_commit_sha or "").strip(),
str(settings.build_commit_ref or "").strip(),
str(settings.build_generated_at or "").strip(),
str(settings.build_package_name or "").strip(),
str(settings.build_checksum or "").strip(),
]
):
return None
return {
"source": str(settings.build_source_label or "").strip() or "env",
"package_name": str(settings.build_package_name or "").strip(),
"generated_at": str(settings.build_generated_at or "").strip(),
"commit_sha": str(settings.build_commit_sha or "").strip(),
"commit_ref": str(settings.build_commit_ref or "").strip(),
"checksum": str(settings.build_checksum or "").strip(),
"manifest_path": str(settings.build_manifest_path or "").strip(),
}
def _build_info_from_manifest() -> dict | None:
manifest_path, source = _discover_manifest_candidate()
if manifest_path is None:
return None
payload = _read_json(manifest_path)
if not payload:
return {
"source": source,
"package_name": "",
"generated_at": "",
"commit_sha": "",
"commit_ref": "",
"checksum": "",
"manifest_path": str(manifest_path),
}
checksum = str(payload.get("checksum") or payload.get("sha256") or "").strip()
return {
"source": source,
"package_name": str(payload.get("package_name") or "").strip(),
"generated_at": str(payload.get("generated_at") or "").strip(),
"commit_sha": str(payload.get("commit_sha") or "").strip(),
"commit_ref": str(payload.get("commit_ref") or "").strip(),
"checksum": checksum,
"manifest_path": str(manifest_path),
}
def _build_info_from_git() -> dict | None:
try:
inside_worktree = subprocess.run(
["git", "-C", str(WORKSPACE_ROOT), "rev-parse", "--is-inside-work-tree"],
check=False,
capture_output=True,
text=True,
timeout=2,
)
except Exception:
return None
if inside_worktree.returncode != 0 or str(inside_worktree.stdout or "").strip() != "true":
return None
def _read_git(*args: str) -> str:
try:
result = subprocess.run(
["git", "-C", str(WORKSPACE_ROOT), *args],
check=False,
capture_output=True,
text=True,
timeout=2,
)
except Exception:
return ""
if result.returncode != 0:
return ""
return str(result.stdout or "").strip()
return {
"source": "git",
"package_name": "",
"generated_at": "",
"commit_sha": _read_git("rev-parse", "--short=12", "HEAD"),
"commit_ref": _read_git("rev-parse", "--abbrev-ref", "HEAD"),
"checksum": "",
"manifest_path": "",
}
def _resolve_build_identity() -> dict:
return (
_build_info_from_env()
or _build_info_from_manifest()
or _build_info_from_git()
or {
"source": "unknown",
"package_name": "",
"generated_at": "",
"commit_sha": "",
"commit_ref": "",
"checksum": "",
"manifest_path": "",
}
)
def _expected_route_paths() -> dict[str, str]:
prefix = str(settings.api_prefix or "/api/v1").rstrip("/")
return {
"ops_contracts": f"{prefix}/ops/contracts",
"ops_contract_detail": f"{prefix}/ops/contracts/{{contract_key}}",
"ops_stack_diagnosis": f"{prefix}/ops/stack-diagnosis",
"ops_node_handover": f"{prefix}/ops/nodes/{{node_code}}/handover",
"ops_node_onboarding": f"{prefix}/ops/nodes/{{node_code}}/onboarding",
"ops_node_onboarding_bootstrap_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/preview",
"ops_node_onboarding_bootstrap_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/execute",
"ops_node_onboarding_acceptance_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/preview",
"ops_node_onboarding_acceptance_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/execute",
"ops_node_onboarding_recovery_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/preview",
"ops_node_onboarding_recovery_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/execute",
"ops_node_scene_log": f"{prefix}/ops/nodes/{{node_code}}/scene-log",
"ops_node_handover_bootstrap_plan": f"{prefix}/ops/nodes/{{node_code}}/handover/bootstrap-plan",
"ops_driver_feed": f"{prefix}/ops/driver-feed",
"ops_codex_brief": f"{prefix}/ops/codex-brief",
"ops_activity_stream": f"{prefix}/ops/activity-stream",
"ops_runbook_resolve": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/resolve",
"ops_runbook_execute": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/execute",
"runtime_build_info": f"{prefix}/runtime/build-info",
}
def _read_text(path: Path) -> str:
try:
return path.read_text(encoding="utf-8", errors="replace")
except Exception:
return ""
def _repository_capabilities() -> dict:
ops_agent_service_text = _read_text(DOMAIN_API_ROOT / "app" / "services" / "ops_agent_service.py")
bootstrap_node_agent_text = _read_text(DOMAIN_API_ROOT / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
return {
"supports_install_command_block": (
"install_command_block" in ops_agent_service_text
and "_candidate_node_agent_install_paths" in ops_agent_service_text
),
"supports_multi_layout_bootstrap": "resolve_project_root" in bootstrap_node_agent_text,
}
def _build_route_surface(route_paths: Iterable[str] | None = None) -> dict:
expected = _expected_route_paths()
normalized_paths = {
str(item).strip()
for item in (list(route_paths) if route_paths is not None else list(_REGISTERED_ROUTE_PATHS))
if str(item).strip()
}
mode = "registered" if normalized_paths else "declared_contract"
flags = {key: (path in normalized_paths if normalized_paths else True) for key, path in expected.items()}
missing_keys = [key for key, available in flags.items() if not available]
return {
"mode": mode,
"registered_paths_total": len(normalized_paths),
"surface_flags": flags,
"expected_paths": expected,
"missing_keys": missing_keys,
"missing_paths": [expected[key] for key in missing_keys],
"surface_complete": len(missing_keys) == 0,
}
def get_runtime_build_info(route_paths: Iterable[str] | None = None) -> dict:
build = _resolve_build_identity()
route_surface = _build_route_surface(route_paths=route_paths)
return {
"source": str(build.get("source") or "").strip() or "unknown",
"package_name": str(build.get("package_name") or "").strip(),
"generated_at": str(build.get("generated_at") or "").strip(),
"commit_sha": str(build.get("commit_sha") or "").strip(),
"commit_ref": str(build.get("commit_ref") or "").strip(),
"checksum": str(build.get("checksum") or "").strip(),
"manifest_path": str(build.get("manifest_path") or "").strip(),
"workspace_root": str(WORKSPACE_ROOT),
"domain_api_root": str(DOMAIN_API_ROOT),
"repository_capabilities": _repository_capabilities(),
"route_surface": route_surface,
}

View File

@@ -147,15 +147,50 @@ def _build_remote_log_lines(
mode: str,
limit: int = 240,
) -> list[str]:
return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)["lines"]
def _build_remote_log_snapshot(
active_job: dict | None,
*,
enabled: bool,
mode: str,
limit: int = 240,
) -> dict:
if not enabled:
return []
return {
"lines": [],
"line_count": 0,
"last_at": "",
"last_line": "",
"source_nodes": [],
"source_node_count": 0,
}
if not active_job:
return []
return {
"lines": [],
"line_count": 0,
"last_at": "",
"last_line": "",
"source_nodes": [],
"source_node_count": 0,
}
events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
if not events:
return []
return {
"lines": [],
"line_count": 0,
"last_at": "",
"last_line": "",
"source_nodes": [],
"source_node_count": 0,
}
lines: list[str] = []
source_nodes: set[str] = set()
source_node_summaries: dict[str, dict] = {}
last_at = ""
last_line = ""
normalized_mode = str(mode or "key").strip().lower()
if normalized_mode not in {"key", "full"}:
normalized_mode = "key"
@@ -180,8 +215,47 @@ def _build_remote_log_lines(
continue
if len(message) > _REMOTE_LOG_MAX_CHARS:
message = f"{message[:_REMOTE_LOG_MAX_CHARS]}..."
lines.append(f"[{created_at}] [{node_code}] {message}")
return lines[-max(1, int(limit or 240)) :]
formatted_line = f"[{created_at}] [{node_code}] {message}"
lines.append(formatted_line)
source_nodes.add(node_code)
node_summary = source_node_summaries.setdefault(
node_code,
{
"node_code": node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
},
)
node_summary["line_count"] += 1
if event_mode == "full":
node_summary["full_line_count"] += 1
else:
node_summary["key_line_count"] += 1
node_summary["last_at"] = created_at
node_summary["last_line"] = formatted_line
last_at = created_at
last_line = formatted_line
sliced_lines = lines[-max(1, int(limit or 240)) :]
sorted_source_node_summaries = sorted(
source_node_summaries.values(),
key=lambda item: (
str(item.get("last_at") or ""),
str(item.get("node_code") or ""),
),
reverse=True,
)
return {
"lines": sliced_lines,
"line_count": len(sliced_lines),
"last_at": last_at,
"last_line": last_line,
"source_nodes": sorted(source_nodes),
"source_node_count": len(source_nodes),
"source_node_summaries": sorted_source_node_summaries,
}
def _resolve_remote_log_lines(
@@ -195,6 +269,17 @@ def _resolve_remote_log_lines(
return _build_remote_log_lines(active_job, enabled=enabled, mode=mode, limit=limit)
def _resolve_remote_log_snapshot(
active_job: dict | None,
runs: list[dict],
*,
enabled: bool,
mode: str,
limit: int = 240,
) -> dict:
return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)
def _load_runtime_state() -> dict:
try:
redis_client = get_redis()
@@ -404,13 +489,14 @@ def get_detect_status() -> dict:
runs = sync_detect_runs(runtime_snapshot, progress, settings_summary, active_job=active_job)
worker_log_sync_enabled = bool(runtime_settings.get("worker_log_sync_enabled", False))
worker_log_sync_mode = str(runtime_settings.get("worker_log_sync_mode", "key") or "key")
remote_log_lines = _resolve_remote_log_lines(
remote_log_snapshot = _resolve_remote_log_snapshot(
active_job,
runs,
enabled=worker_log_sync_enabled,
mode=worker_log_sync_mode,
limit=240,
)
remote_log_lines = list(remote_log_snapshot.get("lines") or [])
dependency_alerts = _extract_dependency_alerts(recent_lines)
append_detect_result_projection_if_changed(
detect={
@@ -465,6 +551,12 @@ def get_detect_status() -> dict:
"recent_warning": recent_proxy_warning,
"log_lines": recent_lines,
"remote_log_lines": remote_log_lines,
"remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0),
"remote_log_last_at": str(remote_log_snapshot.get("last_at") or ""),
"remote_log_last_line": str(remote_log_snapshot.get("last_line") or ""),
"remote_log_nodes": list(remote_log_snapshot.get("source_nodes") or []),
"remote_log_node_count": int(remote_log_snapshot.get("source_node_count", 0) or 0),
"remote_log_node_summaries": list(remote_log_snapshot.get("source_node_summaries") or []),
"runs": runs,
"active_job": active_job,
"worker_log_sync_enabled": worker_log_sync_enabled,

View File

@@ -0,0 +1,278 @@
from __future__ import annotations
from pathlib import Path
from typing import Protocol
from app.core.config import settings
STRUCTURED_ACTIONS = {
"health.snapshot",
"service.status",
"service.restart",
"service.start",
"service.stop",
"logs.collect",
"diagnostics.collect",
"runtime.start_worker",
"runtime.stop_worker",
"runtime.restart_api",
"runtime.start_sync_agent",
"runtime.stop_sync_agent",
}
class CommandRunner(Protocol):
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
...
def supports_structured_action(action: str) -> bool:
return str(action or "").strip() in STRUCTURED_ACTIONS
def trim_output(text: str, limit: int) -> str:
normalized = str(text or "").strip()
if len(normalized) <= int(limit):
return normalized
return normalized[-int(limit):]
def _read_tail_lines(path: Path, *, max_lines: int) -> list[str]:
if not path.exists() or not path.is_file():
return []
with path.open("r", encoding="utf-8", errors="replace") as handle:
return handle.read().splitlines()[-max_lines:]
def _runtime_logs_dir() -> Path:
return Path(__file__).resolve().parents[2] / "runtime" / "logs"
def _service_log_file_candidates(service_name: str, *, service_names: dict[str, str]) -> list[Path]:
normalized_service_name = str(service_name or "").strip()
if not normalized_service_name:
return []
domain_root = Path(settings.domain_root)
runtime_logs_dir = _runtime_logs_dir()
candidates: list[Path] = []
def append_candidate(path: Path) -> None:
if path not in candidates:
candidates.append(path)
if normalized_service_name == (service_names.get("worker") or "domaincheck-worker"):
append_candidate(domain_root / "detect_worker.log")
append_candidate(domain_root / "logs" / "detect_worker.log")
elif normalized_service_name == (service_names.get("api") or "domaincheck-api"):
append_candidate(runtime_logs_dir / "domain-api.stderr.log")
append_candidate(runtime_logs_dir / "domain-api.stdout.log")
append_candidate(domain_root / "logs" / "app.log")
return candidates
def _collect_log_file_fallback(
service_name: str,
*,
lines: int,
service_names: dict[str, str],
) -> tuple[str, list[str]]:
aggregated_lines: list[str] = []
used_paths: list[str] = []
for path in _service_log_file_candidates(service_name, service_names=service_names):
current_lines = _read_tail_lines(path, max_lines=lines)
if not current_lines:
continue
aggregated_lines.extend(current_lines)
used_paths.append(str(path))
if not aggregated_lines:
return "", []
return "\n".join(aggregated_lines[-lines:]), used_paths
def build_service_name_map(
*,
api_service_name: str,
worker_service_name: str,
sync_agent_service_name: str,
node_agent_service_name: str = "domaincheck-node-agent",
) -> dict[str, str]:
return {
"api": str(api_service_name or "").strip() or "domaincheck-api",
"worker": str(worker_service_name or "").strip() or "domaincheck-worker",
"sync_agent": str(sync_agent_service_name or "").strip() or "domaincheck-sync-agent",
"node_agent": str(node_agent_service_name or "").strip() or "domaincheck-node-agent",
}
def service_name_from_payload(payload: dict, *, default_worker_service_name: str) -> str:
service_name = str((payload or {}).get("service_name") or "").strip()
return service_name or str(default_worker_service_name or "").strip() or "domaincheck-worker"
def service_name_for_action(action: str, payload: dict, service_names: dict[str, str]) -> str:
normalized_action = str(action or "").strip()
if normalized_action in {"service.status", "service.restart", "service.start", "service.stop"}:
return service_name_from_payload(payload, default_worker_service_name=service_names.get("worker") or "")
runtime_action_map = {
"runtime.start_worker": service_names.get("worker") or "domaincheck-worker",
"runtime.stop_worker": service_names.get("worker") or "domaincheck-worker",
"runtime.restart_api": service_names.get("api") or "domaincheck-api",
"runtime.start_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
"runtime.stop_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
}
return str(runtime_action_map.get(normalized_action) or "").strip()
def systemctl_action_name(action: str) -> str:
normalized_action = str(action or "").strip()
if normalized_action in {"service.restart", "runtime.restart_api"}:
return "restart"
if normalized_action in {"service.start", "runtime.start_worker", "runtime.start_sync_agent"}:
return "start"
if normalized_action in {"service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"}:
return "stop"
return ""
def execute_structured_action(
action: str,
payload: dict | None,
*,
service_names: dict[str, str],
runner: CommandRunner,
host_context: dict | None = None,
) -> tuple[bool, str, dict]:
normalized_action = str(action or "").strip()
normalized_payload = dict(payload or {})
normalized_service_names = {
str(key or "").strip(): str(value or "").strip()
for key, value in dict(service_names or {}).items()
if str(key or "").strip() and str(value or "").strip()
}
host_details = {
key: str((host_context or {}).get(key) or "").strip()
for key in ("hostname", "ip")
if str((host_context or {}).get(key) or "").strip()
}
if normalized_action == "health.snapshot":
checks: dict[str, dict] = {}
for key, service_name in normalized_service_names.items():
code, stdout, stderr = runner(["systemctl", "is-active", service_name], timeout=15)
checks[key] = {
"service_name": service_name,
"returncode": int(code or 0),
"state": stdout or stderr,
"ok": int(code or 0) == 0 and (stdout or stderr) == "active",
}
result = {"checks": checks}
result.update(host_details)
return True, "health snapshot collected", result
if normalized_action == "service.status":
service_name = service_name_from_payload(
normalized_payload,
default_worker_service_name=normalized_service_names.get("worker") or "",
)
code, stdout, stderr = runner(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
result = {
"service_name": service_name,
"stdout": trim_output(stdout, 12000),
"stderr": trim_output(stderr, 4000),
}
result.update(host_details)
return int(code or 0) == 0, stdout or stderr or f"{service_name} status collected", result
systemctl_action = systemctl_action_name(normalized_action)
if systemctl_action:
service_name = service_name_for_action(normalized_action, normalized_payload, normalized_service_names)
if not service_name:
return False, f"当前动作缺少 service_name: {normalized_action}", {"action": normalized_action}
code, stdout, stderr = runner(["systemctl", systemctl_action, service_name], timeout=45)
result = {
"service_name": service_name,
"systemctl_action": systemctl_action,
"stdout": trim_output(stdout, 12000),
"stderr": trim_output(stderr, 4000),
}
result.update(host_details)
return int(code or 0) == 0, stdout or stderr or f"{service_name} {systemctl_action} completed", result
if normalized_action == "logs.collect":
service_name = service_name_from_payload(
normalized_payload,
default_worker_service_name=normalized_service_names.get("worker") or "",
)
lines = max(20, min(int(normalized_payload.get("lines") or 120), 500))
code, stdout, stderr = runner(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
journal_output = stdout or stderr
fallback_output, fallback_paths = _collect_log_file_fallback(
service_name,
lines=lines,
service_names=normalized_service_names,
)
collection_source = "journal"
effective_output = journal_output
ok = int(code or 0) == 0
if (not ok or not stdout.strip()) and fallback_output:
collection_source = "file_fallback"
effective_output = fallback_output
ok = True
result = {
"service_name": service_name,
"lines": lines,
"collection_source": collection_source,
"fallback_used": bool(collection_source == "file_fallback"),
"fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
"log_paths": fallback_paths,
"journal_returncode": int(code or 0),
"stdout": trim_output(effective_output, 20000),
"stderr": trim_output(stderr, 4000),
}
result.update(host_details)
return ok, effective_output or f"{service_name} logs collected", result
if normalized_action == "diagnostics.collect":
lines = max(20, min(int(normalized_payload.get("lines") or 200), 800))
diagnostics: dict[str, object] = {
"services": {},
"lines": lines,
}
diagnostics.update(host_details)
for key, service_name in normalized_service_names.items():
active_code, active_stdout, active_stderr = runner(["systemctl", "is-active", service_name], timeout=15)
status_code, status_stdout, status_stderr = runner(
["systemctl", "status", service_name, "--no-pager", "-l"],
timeout=45,
)
log_code, log_stdout, log_stderr = runner(
["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"],
timeout=45,
)
log_output = log_stdout or log_stderr
fallback_output, fallback_paths = _collect_log_file_fallback(
service_name,
lines=lines,
service_names=normalized_service_names,
)
log_source = "journal"
if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
log_output = fallback_output
log_source = "file_fallback"
diagnostics["services"][key] = {
"service_name": service_name,
"active_returncode": int(active_code or 0),
"active_state": active_stdout or active_stderr,
"status_returncode": int(status_code or 0),
"status_output": trim_output(status_stdout or status_stderr, 12000),
"log_returncode": int(log_code or 0),
"log_source": log_source,
"log_paths": fallback_paths,
"log_output": trim_output(log_output, 20000),
}
return True, "diagnostics collected", diagnostics
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,17 @@
from __future__ import annotations
OPS_MULTI_REGION_DIR = "domain-api/deploy/multi-region"
def ops_script_path(script_name: str) -> str:
normalized_script_name = str(script_name or "").strip().lstrip("/")
if not normalized_script_name:
return OPS_MULTI_REGION_DIR
return f"{OPS_MULTI_REGION_DIR}/{normalized_script_name}"
def build_bash_command(script_name: str, *args: object) -> str:
command_parts = ["bash", ops_script_path(script_name)]
command_parts.extend(str(arg or "").strip() for arg in args if str(arg or "").strip())
return " ".join(command_parts)

View File

@@ -0,0 +1,79 @@
from __future__ import annotations
from app.services.ops_action_executor_core import STRUCTURED_ACTIONS
_LOCAL_RUNTIME_ONLY_ACTIONS = {
"runtime.push_sync",
"runtime.pull_tasks",
}
_REMOTE_AGENT_ONLY_ACTIONS = {
"delivery.queue.flush",
"delivery.queue.replay",
"delivery.queue.discard",
}
_LOCAL_RUNTIME_ACTIONS = set(STRUCTURED_ACTIONS) | _LOCAL_RUNTIME_ONLY_ACTIONS | {"deploy.release"}
_SSH_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
_REMOTE_AGENT_ACTIONS = set(STRUCTURED_ACTIONS) | _REMOTE_AGENT_ONLY_ACTIONS | {"deploy.release"}
_CONTROL_PLANE_ACTIONS = {
"node.bootstrap",
}
def supports_local_runtime_action(action: str) -> bool:
return str(action or "").strip() in _LOCAL_RUNTIME_ACTIONS
def supports_ssh_action(action: str) -> bool:
return str(action or "").strip() in _SSH_ACTIONS
def supports_remote_agent_action(action: str) -> bool:
return str(action or "").strip() in _REMOTE_AGENT_ACTIONS
def supports_control_plane_action(action: str) -> bool:
return str(action or "").strip() in _CONTROL_PLANE_ACTIONS
def validate_action_execution_mode(
action: str,
execution_mode: str,
*,
target_node_code: str = "",
current_node_code: str = "",
) -> tuple[bool, str]:
normalized_action = str(action or "").strip()
normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
normalized_target_node_code = str(target_node_code or "").strip()
normalized_current_node_code = str(current_node_code or "").strip()
if normalized_mode == "local-runtime":
if not supports_local_runtime_action(normalized_action):
return False, f"{normalized_action} 当前不支持 local-runtime 执行方式。"
if (
normalized_target_node_code
and normalized_current_node_code
and normalized_target_node_code != normalized_current_node_code
):
return False, "local-runtime 仅支持当前控制面本机节点。"
return True, ""
if normalized_mode == "control-plane":
if not supports_control_plane_action(normalized_action):
return False, f"{normalized_action} 当前不支持 control-plane 执行方式。"
return True, ""
if normalized_mode == "ssh":
if not supports_ssh_action(normalized_action):
return False, f"{normalized_action} 当前不支持 ssh 执行方式。"
return True, ""
if normalized_mode == "remote-agent":
if not supports_remote_agent_action(normalized_action):
return False, f"{normalized_action} 当前不支持 remote-agent 执行方式。"
return True, ""
return False, f"未知执行方式: {normalized_mode}"

View File

@@ -0,0 +1,60 @@
from __future__ import annotations
def normalize_execution_modes(raw_modes: object, fallback_mode: str = "remote-agent") -> list[str]:
normalized_modes: list[str] = []
seen: set[str] = set()
fallback = str(fallback_mode or "remote-agent").strip() or "remote-agent"
for item in list(raw_modes or []):
mode = str(item or "").strip()
if not mode or mode in seen:
continue
seen.add(mode)
normalized_modes.append(mode)
if fallback and fallback not in seen:
normalized_modes.insert(0, fallback)
return normalized_modes or [fallback]
def execution_mode_label(execution_mode: str) -> str:
normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
if normalized_mode == "ssh":
return "SSH"
if normalized_mode == "control-plane":
return "控制面"
if normalized_mode == "local-runtime":
return "本机运行时"
if normalized_mode == "remote-agent":
return "远端 Agent"
return normalized_mode
def build_execution_mode_options(raw_modes: object, fallback_mode: str = "remote-agent") -> list[dict]:
return [
{
"value": mode,
"label": execution_mode_label(mode),
}
for mode in normalize_execution_modes(raw_modes, fallback_mode)
]
def decorate_execution_mode_fields(
payload: dict | None = None,
*,
default_key: str = "default_execution_mode",
modes_key: str = "execution_modes",
) -> dict:
normalized_payload = dict(payload or {})
default_mode = str(normalized_payload.get(default_key) or "remote-agent").strip() or "remote-agent"
execution_mode_options = build_execution_mode_options(normalized_payload.get(modes_key) or [], default_mode)
normalized_payload[default_key] = default_mode
normalized_payload[modes_key] = [str(item.get("value") or "").strip() for item in execution_mode_options if str(item.get("value") or "").strip()]
normalized_payload["default_execution_mode_label"] = execution_mode_label(default_mode)
normalized_payload["execution_mode_options"] = execution_mode_options
normalized_payload["execution_mode_labels"] = [
str(item.get("label") or "").strip()
for item in execution_mode_options
if str(item.get("label") or "").strip()
]
return normalized_payload

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,494 @@
from __future__ import annotations
from app.core.config import settings
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.ops_execution_capability_service import validate_action_execution_mode
from app.services.ops_release_service import (
_calculate_total_batches,
_normalize_rollout_policy,
_resolve_rollout_targets,
build_release_rollout_gate,
get_release,
)
_LOW_RISK_ACTIONS = {
"health.snapshot",
"runtime.push_sync",
"runtime.pull_tasks",
"logs.collect",
"diagnostics.collect",
"delivery.queue.flush",
}
_MEDIUM_RISK_ACTIONS = {
"runtime.start_worker",
"runtime.start_sync_agent",
"service.status",
"health.check",
"delivery.queue.replay",
}
_HIGH_RISK_ACTIONS = {
"runtime.stop_worker",
"runtime.stop_sync_agent",
"runtime.restart_api",
"service.restart",
"deploy.release",
"delivery.queue.discard",
}
_CRITICAL_RISK_ACTIONS = {
"deploy.rollback",
"node.bootstrap",
"cluster.reconfigure",
}
def _normalize_node_code_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n").strip()
if not text:
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def _dedupe_text_items(items: list[str]) -> list[str]:
return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip()))
def _risk_level_for_action(action: str) -> str:
if action in _LOW_RISK_ACTIONS:
return "low"
if action in _HIGH_RISK_ACTIONS:
return "high"
if action in _CRITICAL_RISK_ACTIONS:
return "critical"
return "medium"
def _compact_target_node(item: dict) -> dict:
return {
"node_code": str(item.get("node_code") or ""),
"region": str(item.get("region") or ""),
"role": str(item.get("role") or ""),
"status": str(item.get("status") or ""),
"current_load": int(item.get("current_load", 0) or 0),
"is_effective_worker": bool(item.get("is_effective_worker", False)),
"detect_participating": bool(item.get("detect_participating", False)),
"is_enabled": bool(item.get("is_enabled", True)),
"deploy_channel": str(item.get("deploy_channel") or ""),
}
def _preview_action_payload_guardrails(
action: str,
payload: dict,
*,
execution_mode: str,
target_nodes_total: int,
) -> dict:
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if action == "deploy.release":
release_version = str(payload.get("release_version") or "").strip()
artifact_url = str(payload.get("artifact_url") or "").strip()
if not release_version:
blocking_reasons.append("deploy.release 缺少 release_version当前不能创建正式发布任务。")
if not artifact_url:
blocking_reasons.append("deploy.release 缺少 artifact_url当前不能创建正式发布任务。")
if execution_mode not in {"remote-agent", "ssh"}:
blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。")
if target_nodes_total > 1:
recommendations.append("建议先单节点 canary 验证发布任务,再逐步扩到更多节点。")
if action == "node.bootstrap":
if execution_mode != "control-plane":
blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。")
if target_nodes_total > 1:
recommendations.append("接管动作建议按单节点节奏推进,先确认首台节点接入成功后再继续放量。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
return {
"warnings": _dedupe_text_items(warnings),
"blocking_reasons": _dedupe_text_items(blocking_reasons),
"approval_reasons": _dedupe_text_items(approval_reasons),
"recommendations": _dedupe_text_items(recommendations),
}
def _preview_single_node_policy(
action: str,
*,
target_node_code: str,
cluster: dict,
payload: dict | None = None,
execution_mode: str = "remote-agent",
target_type: str = "node",
include_payload_guardrails: bool = True,
) -> dict:
payload = dict(payload or {})
nodes = list(cluster.get("nodes") or [])
summary = cluster.get("summary") or {}
target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
execution_supported, execution_reason = validate_action_execution_mode(
action,
execution_mode,
target_node_code=target_node_code,
current_node_code=settings.node_code,
)
if not execution_supported and execution_reason:
blocking_reasons.append(execution_reason)
if target_type == "node" and target_node_code and not target_node:
warnings.append("目标节点当前未出现在集群快照中,可能尚未注册或已离线。")
node_status = str(target_node.get("status") or "")
node_role = str(target_node.get("role") or "")
node_current_load = int(target_node.get("current_load", 0) or 0)
node_effective_worker = bool(target_node.get("is_effective_worker", False))
node_detect_participating = bool(target_node.get("detect_participating", False))
if target_node:
if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("目标节点正在参与检测,需先迁移负载或人工确认后再执行。")
if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
approval_reasons.append("目标节点是 control 节点,建议强制走审批或维护窗口。")
if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("当前有效执行节点数不足 2执行该动作可能导致检测能力中断。")
if node_current_load > 0:
warnings.append(f"目标节点当前负载为 {node_current_load},建议优先观察或转移任务。")
if include_payload_guardrails:
payload_guardrails = _preview_action_payload_guardrails(
action,
payload,
execution_mode=execution_mode,
target_nodes_total=1,
)
warnings.extend(payload_guardrails.get("warnings") or [])
blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(payload_guardrails.get("approval_reasons") or [])
recommendations.extend(payload_guardrails.get("recommendations") or [])
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": target_type,
"target_node_code": target_node_code,
"execution_mode": execution_mode,
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_node": {
"node_code": str(target_node.get("node_code") or ""),
"region": str(target_node.get("region") or ""),
"role": node_role,
"status": node_status,
"current_load": node_current_load,
"is_effective_worker": node_effective_worker,
"detect_participating": node_detect_participating,
} if target_node else {},
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict:
target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes"))
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
summary = cluster.get("summary") or {}
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
if not target_node_codes:
blocking_reasons.append("当前没有可预检的目标节点。")
shared_guardrails = _preview_action_payload_guardrails(
action,
action_payload,
execution_mode=execution_mode,
target_nodes_total=len(target_node_codes),
)
warnings.extend(shared_guardrails.get("warnings") or [])
blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or [])
approval_reasons.extend(shared_guardrails.get("approval_reasons") or [])
recommendations.extend(shared_guardrails.get("recommendations") or [])
node_previews = [
_preview_single_node_policy(
action,
target_node_code=node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type="batch",
include_payload_guardrails=False,
)
for node_code in target_node_codes
]
blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))]
approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))]
warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])]
missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")]
target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"]
if blocked_nodes:
blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断,需先处理后再批量执行。")
if approval_nodes:
approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。")
if warning_nodes:
warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。")
if missing_nodes:
warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。")
if target_control_nodes and len(target_node_codes) > 1:
recommendations.append("建议把 control 节点放到批次尾部,优先在 worker 或空闲节点验证。")
warnings = _dedupe_text_items(warnings)
blocking_reasons = _dedupe_text_items(blocking_reasons)
approval_reasons = _dedupe_text_items(approval_reasons)
recommendations = _dedupe_text_items(recommendations)
return {
"action": action,
"target_type": "batch",
"execution_mode": execution_mode,
"risk_level": _risk_level_for_action(action),
"approval_required": bool(approval_reasons),
"blocked": bool(blocking_reasons),
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": len(target_node_codes),
"blocked_nodes": len(blocked_nodes),
"approval_nodes": len(approval_nodes),
"warning_nodes": len(warning_nodes),
"missing_nodes": len(missing_nodes),
},
"target_nodes": [
{
**(item.get("target_node") or {}),
"node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""),
"risk_level": str(item.get("risk_level") or ""),
"approval_required": bool(item.get("approval_required", False)),
"blocked": bool(item.get("blocked", False)),
"blocking_reasons": list(item.get("blocking_reasons") or []),
"approval_reasons": list(item.get("approval_reasons") or []),
"warnings": list(item.get("warnings") or []),
}
for item in node_previews
],
"cluster_guardrails": {
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
"busy_nodes": list(summary.get("busy_nodes") or []),
},
}
def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict:
selector = payload.get("target_selector") or {}
input_policy = payload.get("policy") or {}
release_id = int(payload.get("release_id") or 0)
input_release = dict(payload.get("release") or {})
release = input_release or (get_release(release_id) if release_id > 0 else {})
target_nodes = _resolve_rollout_targets(selector)
normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes))
execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
summary = cluster.get("summary") or {}
release_gate = build_release_rollout_gate(
release,
target_nodes,
execution_mode=execution_mode,
)
operational_readiness = dict(release_gate.get("operational_readiness") or {})
risk_level = _risk_level_for_action(action)
warnings: list[str] = []
blocking_reasons: list[str] = []
approval_reasons: list[str] = []
recommendations: list[str] = []
target_count = len(target_nodes)
first_batch_size = int(normalized_policy.get("first_batch_size") or 0)
batch_size = int(normalized_policy.get("batch_size") or 0)
batches_total = _calculate_total_batches(target_count, normalized_policy)
first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else []
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0)
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"]
target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"]
target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))]
first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))]
first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"]
busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0]
participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))]
offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}]
if target_count <= 0:
blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。")
if release_id > 0 and not input_release and not release:
blocking_reasons.append("指定的 Release 不存在,当前无法预检该 rollout。")
gate_status = str(release_gate.get("status") or "").strip()
if gate_status in {"release_not_ready", "artifact_missing"}:
warnings.append(str(release_gate.get("summary") or "").strip())
if offline_targets:
warnings.append(f"命中了 {len(offline_targets)} 个非在线节点,若保留 only_online=false发布时可能出现排队或失败。")
if busy_targets:
warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点,建议优先确认任务是否可中断。")
if participating_targets:
warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点,建议优先滚动到空闲节点。")
if target_control_nodes:
approval_reasons.append("本次 rollout 覆盖 control 节点,建议强制审批,并把 control 放在最后一批。")
if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes:
blocking_reasons.append("第一批会覆盖全部有效执行节点,检测能力可能瞬时归零。请缩小 first_batch_size。")
if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes:
blocking_reasons.append("第一批会覆盖全部在线控制面,运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。")
if first_batch_size > 1 and target_control_nodes:
warnings.append("当前第一批大于 1且目标中包含 control 节点;默认建议先 worker、后 control。")
if action in _CRITICAL_RISK_ACTIONS:
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
elif action in _HIGH_RISK_ACTIONS:
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
blocking_reasons.extend(list(release_gate.get("blocking_reasons") or []))
warnings.extend(list(release_gate.get("warning_reasons") or []))
recommendations.extend(list(release_gate.get("recommendations") or []))
readiness_summary = dict(operational_readiness.get("summary") or {})
if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count:
approval_reasons.append("本次目标中存在巡检未完全通过的节点,建议先完成标准巡检并经人工确认后再发布。")
if target_count > 1 and first_batch_size > 1:
recommendations.append("正式环境建议 first_batch_size=1从单节点 canary 开始。")
if target_control_nodes:
recommendations.append("建议把 control 节点放在 rollout 末尾,并确保 worker 已先通过健康检查。")
if busy_targets or participating_targets:
recommendations.append("建议先等参与检测的节点降为空闲,再推进该批次。")
warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip()))
blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip()))
approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip()))
recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip()))
approval_required = bool(approval_reasons)
blocked = bool(blocking_reasons)
return {
"action": action,
"target_type": "rollout",
"risk_level": risk_level,
"approval_required": approval_required,
"blocked": blocked,
"blocking_reasons": blocking_reasons,
"approval_reasons": approval_reasons,
"warnings": warnings,
"recommendations": recommendations,
"target_summary": {
"nodes_total": target_count,
"worker_nodes": len(target_worker_nodes),
"control_nodes": len(target_control_nodes),
"effective_worker_nodes": len(target_effective_workers),
"busy_nodes": len(busy_targets),
"participating_nodes": len(participating_targets),
"offline_nodes": len(offline_targets),
},
"batch_plan": {
"first_batch_size": first_batch_size,
"batch_size": batch_size,
"batches_total": batches_total,
"first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets],
"remaining_after_first_batch": max(target_count - len(first_batch_targets), 0),
},
"cluster_guardrails": {
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"online_control_nodes": online_control_nodes,
"busy_nodes": list(summary.get("busy_nodes") or []),
},
"release": release,
"release_gate": release_gate,
"target_nodes": [_compact_target_node(item) for item in target_nodes],
"operational_readiness": operational_readiness,
}
def preview_ops_job_policy(payload: dict) -> dict:
action = str(payload.get("action") or "").strip()
target_type = str(payload.get("target_type") or "node").strip() or "node"
target_node_code = str(payload.get("target_node_code") or "").strip()
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
action_payload = dict(payload.get("payload") or {})
cluster = get_cluster_snapshot()
if target_type == "rollout":
return _preview_rollout_policy(action, payload, cluster)
if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []):
return _preview_node_batch_policy(action, payload, cluster)
return _preview_single_node_policy(
action,
target_node_code=target_node_code,
cluster=cluster,
payload=action_payload,
execution_mode=execution_mode,
target_type=target_type,
include_payload_guardrails=True,
)

View File

@@ -0,0 +1,475 @@
from __future__ import annotations
import hashlib
import inspect
import json
import shutil
import tarfile
import textwrap
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
def normalize_text_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n").strip()
if not text:
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def coerce_bool(raw_value: object, default: bool = False) -> bool:
if raw_value is None:
return bool(default)
if isinstance(raw_value, bool):
return raw_value
if isinstance(raw_value, (int, float)):
return bool(raw_value)
normalized = str(raw_value or "").strip().lower()
if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
return True
if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
return False
return bool(default) if normalized == "" else bool(normalized)
def normalize_release_health_check_services(
payload: dict,
restart_services: list[str],
*,
default_api_service_name: str,
) -> list[str]:
raw_health_check_services = payload.get("health_check_services")
if raw_health_check_services is None:
health_check_service_source = restart_services or [default_api_service_name]
else:
health_check_service_source = raw_health_check_services
return normalize_text_list(health_check_service_source)
def collect_service_state(run_command, service_name: str) -> dict:
code, stdout, stderr = run_command(["systemctl", "is-active", service_name], timeout=15)
state = stdout or stderr
return {
"service_name": service_name,
"returncode": int(code or 0),
"state": state,
"ok": int(code or 0) == 0 and state == "active",
}
def check_health_url(url: str, timeout: int, *, user_agent: str, urlopen_func) -> dict:
request = urllib.request.Request(
url=str(url).strip(),
headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
method="GET",
)
try:
with urlopen_func(request, timeout=timeout) as response:
body = response.read(4000).decode("utf-8", errors="ignore")
status_code = int(getattr(response, "status", 0) or response.getcode() or 0)
return {
"url": str(url).strip(),
"ok": 200 <= status_code < 400,
"http_status": status_code,
"body_preview": body[-1000:],
}
except urllib.error.HTTPError as exc:
body = exc.read().decode("utf-8", errors="ignore") if exc.fp else ""
return {
"url": str(url).strip(),
"ok": False,
"http_status": int(exc.code or 0),
"body_preview": body[-1000:],
"error": str(exc),
}
except Exception as exc:
return {
"url": str(url).strip(),
"ok": False,
"http_status": 0,
"body_preview": "",
"error": str(exc),
}
def run_release_health_checks(
*,
urls: list[str],
services: list[str],
timeout: int,
retries: int,
interval_seconds: int,
run_command,
user_agent: str,
urlopen_func,
) -> tuple[bool, dict]:
normalized_urls = [str(item).strip() for item in urls if str(item).strip()]
normalized_services = [str(item).strip() for item in services if str(item).strip()]
attempts: list[dict] = []
total_attempts = max(1, retries + 1)
for attempt_index in range(1, total_attempts + 1):
service_results = [collect_service_state(run_command, service_name) for service_name in normalized_services]
url_results = [
check_health_url(url, timeout=timeout, user_agent=user_agent, urlopen_func=urlopen_func)
for url in normalized_urls
]
services_ok = all(bool(item.get("ok", False)) for item in service_results) if service_results else True
urls_ok = all(bool(item.get("ok", False)) for item in url_results) if url_results else True
attempt_payload = {
"attempt": attempt_index,
"services": service_results,
"urls": url_results,
"ok": services_ok and urls_ok,
}
attempts.append(attempt_payload)
if attempt_payload["ok"]:
return True, {
"ok": True,
"attempts": attempts,
"services_checked": normalized_services,
"urls_checked": normalized_urls,
}
if attempt_index < total_attempts:
time.sleep(max(0, interval_seconds))
return False, {
"ok": False,
"attempts": attempts,
"services_checked": normalized_services,
"urls_checked": normalized_urls,
}
def safe_extract_tar(archive: tarfile.TarFile, target_dir: Path) -> None:
target_dir_resolved = target_dir.resolve()
members = archive.getmembers()
for member in members:
member_path = (target_dir / member.name).resolve()
if not str(member_path).startswith(str(target_dir_resolved)):
raise RuntimeError(f"unsafe archive member: {member.name}")
try:
archive.extractall(target_dir, members=members, filter="data")
except TypeError:
archive.extractall(target_dir, members=members)
def execute_release_action(
payload: dict,
*,
run_command,
default_api_service_name: str,
event_callback=None,
urlopen_func=None,
user_agent: str = "domaincheck-ops/0.1",
) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
event_callback = event_callback or (lambda event_type, message, level="info", payload=None: None)
urlopen_func = urlopen_func or urllib.request.urlopen
release_version = str(normalized_payload.get("release_version") or "").strip()
artifact_url = str(normalized_payload.get("artifact_url") or "").strip()
checksum = str(normalized_payload.get("checksum") or "").strip().lower()
install_root = Path(str(normalized_payload.get("install_root") or "/opt/domaincheck")).resolve()
switch_current = coerce_bool(normalized_payload.get("switch_current", True), default=True)
restart_services = normalize_text_list(normalized_payload.get("restart_services"))
health_check_urls = normalize_text_list(normalized_payload.get("health_check_urls"))
health_check_services = normalize_release_health_check_services(
normalized_payload,
restart_services,
default_api_service_name=default_api_service_name,
)
health_check_timeout_seconds = max(2, int(normalized_payload.get("health_check_timeout_seconds") or 10))
health_check_retries = max(0, int(normalized_payload.get("health_check_retries") or 2))
health_check_interval_seconds = max(0, int(normalized_payload.get("health_check_interval_seconds") or 2))
rollback_on_failure = coerce_bool(normalized_payload.get("rollback_on_failure", True), default=True)
if not release_version:
return False, "release_version missing", {}
if not artifact_url:
return False, "artifact_url missing", {}
downloads_dir = install_root / "downloads"
releases_dir = install_root / "releases"
target_dir = releases_dir / release_version
temp_dir = releases_dir / f"{release_version}.tmp"
artifact_path = downloads_dir / f"{release_version}.tar.gz"
current_link = install_root / "current"
previous_current_target = ""
downloads_dir.mkdir(parents=True, exist_ok=True)
releases_dir.mkdir(parents=True, exist_ok=True)
if current_link.exists():
try:
previous_current_target = str(current_link.resolve(strict=True))
except Exception:
try:
previous_current_target = str(current_link.resolve())
except Exception:
previous_current_target = ""
event_callback(
"deploy_download_started",
f"开始下载发布包: {release_version}",
payload={"artifact_url": artifact_url},
)
request = urllib.request.Request(
url=artifact_url,
headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
method="GET",
)
with urlopen_func(request, timeout=120) as response:
artifact_bytes = response.read()
artifact_path.write_bytes(artifact_bytes)
event_callback(
"deploy_download_completed",
f"发布包下载完成: {release_version}",
payload={"artifact_path": str(artifact_path), "size_bytes": len(artifact_bytes)},
)
calculated_checksum = hashlib.sha256(artifact_bytes).hexdigest().lower()
if checksum and checksum != calculated_checksum:
return False, "release checksum mismatch", {
"expected_checksum": checksum,
"calculated_checksum": calculated_checksum,
}
event_callback(
"deploy_checksum_verified",
f"发布包校验通过: {release_version}",
payload={"checksum": calculated_checksum},
)
if temp_dir.exists():
shutil.rmtree(temp_dir, ignore_errors=True)
temp_dir.mkdir(parents=True, exist_ok=True)
with tarfile.open(artifact_path, "r:gz") as archive:
safe_extract_tar(archive, temp_dir)
if target_dir.exists():
extracted_target = target_dir
shutil.rmtree(temp_dir, ignore_errors=True)
else:
temp_dir.rename(target_dir)
extracted_target = target_dir
meta_path = extracted_target / ".release-meta.json"
meta_path.write_text(
json.dumps(
{
"release_version": release_version,
"artifact_url": artifact_url,
"checksum": calculated_checksum,
"deployed_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
if switch_current:
if current_link.is_symlink() or current_link.is_file():
current_link.unlink(missing_ok=True)
elif current_link.exists():
raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
current_link.symlink_to(extracted_target)
event_callback(
"deploy_current_switched",
f"current 已切换到 {release_version}",
payload={"current_link": str(current_link), "target": str(extracted_target)},
)
restarted: list[dict] = []
for service_name in restart_services:
normalized_service_name = str(service_name or "").strip()
if not normalized_service_name:
continue
event_callback(
"deploy_service_restart",
f"重启服务: {normalized_service_name}",
payload={"service_name": normalized_service_name},
)
code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
restarted.append(
{
"service_name": normalized_service_name,
"returncode": int(code or 0),
"stdout": stdout,
"stderr": stderr,
}
)
if int(code or 0) != 0:
return False, f"restart failed: {normalized_service_name}", {
"release_version": release_version,
"release_dir": str(extracted_target),
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"previous_current_target": previous_current_target,
"restart_results": restarted,
}
health_ok, health_result = run_release_health_checks(
urls=health_check_urls,
services=health_check_services,
timeout=health_check_timeout_seconds,
retries=health_check_retries,
interval_seconds=health_check_interval_seconds,
run_command=run_command,
user_agent=user_agent,
urlopen_func=urlopen_func,
)
if not health_ok:
rollback_result = {
"attempted": rollback_on_failure,
"restored_to": previous_current_target,
"restart_results": [],
"health_result": health_result,
}
event_callback(
"deploy_health_failed",
f"发布健康检查失败: {release_version}",
level="error",
payload=health_result,
)
if rollback_on_failure and previous_current_target:
if current_link.is_symlink() or current_link.is_file():
current_link.unlink(missing_ok=True)
elif current_link.exists():
raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
current_link.symlink_to(Path(previous_current_target))
for service_name in restart_services:
normalized_service_name = str(service_name or "").strip()
if not normalized_service_name:
continue
code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
rollback_result["restart_results"].append(
{
"service_name": normalized_service_name,
"returncode": int(code or 0),
"stdout": stdout,
"stderr": stderr,
}
)
rollback_result["post_rollback_health"] = run_release_health_checks(
urls=health_check_urls,
services=health_check_services,
timeout=health_check_timeout_seconds,
retries=0,
interval_seconds=0,
run_command=run_command,
user_agent=user_agent,
urlopen_func=urlopen_func,
)[1]
event_callback(
"deploy_rollback_completed",
f"已回滚到旧版本: {previous_current_target}",
level="warning",
payload=rollback_result,
)
return False, "release health check failed", {
"release_version": release_version,
"release_dir": str(extracted_target),
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"current_link": str(current_link),
"previous_current_target": previous_current_target,
"restart_results": restarted,
"health_check": health_result,
"rollback": rollback_result,
}
event_callback(
"deploy_health_passed",
f"发布健康检查通过: {release_version}",
payload=health_result,
)
return True, "release deployed", {
"release_version": release_version,
"release_dir": str(extracted_target),
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"current_link": str(current_link),
"previous_current_target": previous_current_target,
"restart_results": restarted,
"health_check": health_result,
}
def build_remote_release_action_script(
payload: dict,
*,
default_api_service_name: str = "domaincheck-api",
user_agent: str = "domaincheck-ssh/0.1",
) -> str:
helper_functions = [
normalize_text_list,
coerce_bool,
normalize_release_health_check_services,
collect_service_state,
check_health_url,
run_release_health_checks,
safe_extract_tar,
execute_release_action,
]
helper_source = "\n\n".join(
textwrap.dedent(inspect.getsource(func)).strip("\n")
for func in helper_functions
)
return f"""from __future__ import annotations
import hashlib
import json
import shutil
import tarfile
import time
import urllib.error
import urllib.request
from datetime import datetime
from pathlib import Path
{helper_source}
PAYLOAD = {json.dumps(dict(payload or {{}}), ensure_ascii=False)}
DEFAULT_API_SERVICE_NAME = {json.dumps(str(default_api_service_name or 'domaincheck-api'), ensure_ascii=False)}
USER_AGENT = {json.dumps(str(user_agent or 'domaincheck-ssh/0.1'), ensure_ascii=False)}
def _run(command, timeout=60):
import subprocess
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
def _event_callback(event_type, message, level="info", payload=None):
return None
def main():
try:
ok, message, result = execute_release_action(
PAYLOAD,
run_command=_run,
default_api_service_name=DEFAULT_API_SERVICE_NAME,
event_callback=_event_callback,
urlopen_func=urllib.request.urlopen,
user_agent=USER_AGENT,
)
except Exception as exc:
ok, message, result = False, str(exc), {{"exception": str(exc), "action": "deploy.release"}}
print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
raise SystemExit(0 if ok else 1)
if __name__ == "__main__":
main()
"""

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,420 @@
from __future__ import annotations
import json
import subprocess
from app.core.config import settings
from app.services.ops_action_executor_core import (
STRUCTURED_ACTIONS,
build_service_name_map,
execute_structured_action,
trim_output,
)
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
_SSH_SUPPORTED_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
_SSH_CONNECT_TIMEOUT_SECONDS = 12
_SSH_REMOTE_TIMEOUT_SECONDS = {
"health.snapshot": 30,
"service.status": 45,
"service.restart": 45,
"service.start": 45,
"service.stop": 45,
"logs.collect": 45,
"diagnostics.collect": 90,
"runtime.start_worker": 45,
"runtime.stop_worker": 45,
"runtime.restart_api": 45,
"runtime.start_sync_agent": 45,
"runtime.stop_sync_agent": 45,
"deploy.release": 180,
}
def supports_ssh_execution(action: str) -> bool:
return str(action or "").strip() in _SSH_SUPPORTED_ACTIONS
def execute_local_support_action(action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_action = str(action or "").strip()
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
return False, f"当前未实现本机支持动作: {normalized_action}", {}
if normalized_action == "deploy.release":
return execute_release_action(
dict(payload or {}),
run_command=_run_local_command,
default_api_service_name=settings.api_service_name,
user_agent="domaincheck-local-runtime/0.1",
)
return execute_structured_action(
normalized_action,
dict(payload or {}),
service_names=_service_name_map(),
runner=_run_local_command,
)
def execute_ssh_action(node: dict, action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_action = str(action or "").strip()
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
return False, f"当前 SSH 执行器暂不支持动作: {normalized_action}", {}
ssh_host = str(node.get("ssh_host") or "").strip()
ssh_user = str(node.get("ssh_user") or "").strip()
ssh_port = max(1, int(node.get("ssh_port") or 22))
node_code = str(node.get("node_code") or "").strip()
if not ssh_host or not ssh_user:
return False, "目标节点缺少 SSH 主机或用户信息", {
"node_code": node_code,
"ssh_host": ssh_host,
"ssh_user": ssh_user,
"ssh_port": ssh_port,
}
normalized_payload = dict(payload or {})
service_names = _service_name_map()
if normalized_action == "deploy.release":
remote_script = build_remote_release_action_script(
normalized_payload,
default_api_service_name=service_names.get("api") or "domaincheck-api",
user_agent="domaincheck-ssh/0.1",
)
else:
remote_script = _build_remote_python_script(
action=normalized_action,
payload=normalized_payload,
service_names=service_names,
)
remote_command = "\n".join(
[
"set -euo pipefail",
"if command -v python3 >/dev/null 2>&1; then",
" PYTHON_BIN=python3",
"elif command -v python >/dev/null 2>&1; then",
" PYTHON_BIN=python",
"else",
' echo "{\\"ok\\": false, \\"message\\": \\"python interpreter missing on remote node\\", \\"result\\": {\\"executor\\": \\"ssh\\"}}"',
" exit 127",
"fi",
'"${PYTHON_BIN}" - <<\'PY\'',
remote_script.rstrip("\n"),
"PY",
]
)
ssh_command = [
"ssh",
"-o",
"BatchMode=yes",
"-o",
"PreferredAuthentications=publickey",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
"-p",
str(ssh_port),
f"{ssh_user}@{ssh_host}",
remote_command,
]
timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS
completed = subprocess.run(
ssh_command,
capture_output=True,
text=True,
timeout=timeout_seconds,
)
transport = {
"executor": "ssh",
"node_code": node_code,
"ssh_host": ssh_host,
"ssh_user": ssh_user,
"ssh_port": ssh_port,
"action": normalized_action,
"returncode": int(completed.returncode or 0),
}
parsed_ok, parsed_payload = _extract_json_from_output(completed.stdout or "")
if parsed_ok:
ok = bool(parsed_payload.get("ok", False))
message = str(parsed_payload.get("message") or "").strip() or ("SSH 动作执行成功" if ok else "SSH 动作执行失败")
result = dict(parsed_payload.get("result") or {})
result["transport"] = {
**transport,
"stdout_preview": trim_output(completed.stdout, 4000),
"stderr_preview": trim_output(completed.stderr, 2000),
}
if completed.returncode != 0 and ok:
ok = False
message = f"SSH 命令返回码异常: {completed.returncode}"
return ok, message, result
stderr_preview = trim_output(completed.stderr, 4000)
stdout_preview = trim_output(completed.stdout, 4000)
message = stderr_preview or stdout_preview or f"SSH 执行失败,返回码 {completed.returncode}"
return False, message, {
"executor": "ssh",
"action": normalized_action,
"transport": {
**transport,
"stdout_preview": stdout_preview,
"stderr_preview": stderr_preview,
},
}
def _service_name_map() -> dict[str, str]:
return build_service_name_map(
api_service_name=settings.api_service_name,
worker_service_name=settings.worker_service_name,
sync_agent_service_name=settings.sync_agent_service_name,
)
def _run_local_command(command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
def _build_remote_python_script(*, action: str, payload: dict, service_names: dict[str, str]) -> str:
return f"""import json
import subprocess
import sys
from pathlib import Path
SERVICE_NAMES = {json.dumps(service_names, ensure_ascii=False)}
ACTION = {json.dumps(str(action or '').strip(), ensure_ascii=False)}
PAYLOAD = {json.dumps(dict(payload or {}), ensure_ascii=False)}
def trim_output(text, limit):
value = str(text or "").strip()
if len(value) <= int(limit):
return value
return value[-int(limit):]
def run(cmd, timeout=60):
completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
def read_tail_lines(path, max_lines):
file_path = Path(path)
if not file_path.exists() or not file_path.is_file():
return []
with file_path.open("r", encoding="utf-8", errors="replace") as handle:
return handle.read().splitlines()[-int(max_lines):]
def candidate_domain_roots():
roots = []
for candidate in (
Path.cwd() / "domainCheck",
Path("/opt/domaincheck/domainCheck"),
Path("/www/wwwroot/getDomain/domainCheck"),
):
if candidate not in roots:
roots.append(candidate)
return roots
def candidate_runtime_logs_dirs():
dirs = []
for candidate in (
Path.cwd() / "domain-api" / "runtime" / "logs",
Path("/opt/domaincheck/domain-api/runtime/logs"),
Path("/www/wwwroot/getDomain/domain-api/runtime/logs"),
):
if candidate not in dirs:
dirs.append(candidate)
return dirs
def service_log_file_candidates(service_name):
normalized_service_name = str(service_name or "").strip()
if not normalized_service_name:
return []
candidates = []
def append_candidate(path):
if path not in candidates:
candidates.append(path)
if normalized_service_name == (SERVICE_NAMES.get("worker") or "domaincheck-worker"):
for domain_root in candidate_domain_roots():
append_candidate(domain_root / "detect_worker.log")
append_candidate(domain_root / "logs" / "detect_worker.log")
elif normalized_service_name == (SERVICE_NAMES.get("api") or "domaincheck-api"):
for runtime_logs_dir in candidate_runtime_logs_dirs():
append_candidate(runtime_logs_dir / "domain-api.stderr.log")
append_candidate(runtime_logs_dir / "domain-api.stdout.log")
for domain_root in candidate_domain_roots():
append_candidate(domain_root / "logs" / "app.log")
return candidates
def collect_log_file_fallback(service_name, lines):
aggregated_lines = []
used_paths = []
for path in service_log_file_candidates(service_name):
current_lines = read_tail_lines(path, lines)
if not current_lines:
continue
aggregated_lines.extend(current_lines)
used_paths.append(str(path))
if not aggregated_lines:
return "", []
return "\\n".join(aggregated_lines[-int(lines):]), used_paths
def service_name_from_payload(payload):
value = str((payload or {{}}).get("service_name") or "").strip()
return value or SERVICE_NAMES.get("worker") or "domaincheck-worker"
def service_name_for_action(action, payload):
if action in ("service.status", "service.restart", "service.start", "service.stop"):
return service_name_from_payload(payload)
runtime_action_map = {{
"runtime.start_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
"runtime.stop_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
"runtime.restart_api": SERVICE_NAMES.get("api") or "domaincheck-api",
"runtime.start_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
"runtime.stop_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
}}
return str(runtime_action_map.get(action) or "").strip()
def systemctl_action_name(action):
if action in ("service.restart", "runtime.restart_api"):
return "restart"
if action in ("service.start", "runtime.start_worker", "runtime.start_sync_agent"):
return "start"
if action in ("service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"):
return "stop"
return ""
def execute(action, payload):
if action == "health.snapshot":
checks = {{}}
for key, service_name in SERVICE_NAMES.items():
code, stdout, stderr = run(["systemctl", "is-active", service_name], timeout=15)
checks[key] = {{
"service_name": service_name,
"returncode": int(code or 0),
"state": stdout or stderr,
"ok": int(code or 0) == 0 and (stdout or stderr) == "active",
}}
return True, "health snapshot collected", {{"checks": checks}}
if action == "service.status":
service_name = service_name_from_payload(payload)
code, stdout, stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
result = {{
"service_name": service_name,
"stdout": trim_output(stdout, 12000),
"stderr": trim_output(stderr, 4000),
}}
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} status collected", result
systemctl_action = systemctl_action_name(action)
if systemctl_action:
service_name = service_name_for_action(action, payload)
if not service_name:
return False, f"missing service name for action: {{action}}", {{"action": action}}
code, stdout, stderr = run(["systemctl", systemctl_action, service_name], timeout=45)
result = {{
"service_name": service_name,
"systemctl_action": systemctl_action,
"stdout": trim_output(stdout, 12000),
"stderr": trim_output(stderr, 4000),
}}
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} {{systemctl_action}} completed", result
if action == "logs.collect":
service_name = service_name_from_payload(payload)
lines = max(20, min(int((payload or {{}}).get("lines") or 120), 500))
code, stdout, stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
journal_output = stdout or stderr
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
collection_source = "journal"
effective_output = journal_output
ok = int(code or 0) == 0
if (not ok or not stdout.strip()) and fallback_output:
collection_source = "file_fallback"
effective_output = fallback_output
ok = True
result = {{
"service_name": service_name,
"lines": lines,
"collection_source": collection_source,
"fallback_used": bool(collection_source == "file_fallback"),
"fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
"log_paths": fallback_paths,
"journal_returncode": int(code or 0),
"stdout": trim_output(effective_output, 20000),
"stderr": trim_output(stderr, 4000),
}}
return ok, effective_output or f"{{service_name}} logs collected", result
if action == "diagnostics.collect":
lines = max(20, min(int((payload or {{}}).get("lines") or 200), 800))
diagnostics = {{
"services": {{}},
"lines": lines,
}}
for key, service_name in SERVICE_NAMES.items():
active_code, active_stdout, active_stderr = run(["systemctl", "is-active", service_name], timeout=15)
status_code, status_stdout, status_stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
log_code, log_stdout, log_stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
log_output = log_stdout or log_stderr
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
log_source = "journal"
if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
log_output = fallback_output
log_source = "file_fallback"
diagnostics["services"][key] = {{
"service_name": service_name,
"active_returncode": int(active_code or 0),
"active_state": active_stdout or active_stderr,
"status_returncode": int(status_code or 0),
"status_output": trim_output(status_stdout or status_stderr, 12000),
"log_returncode": int(log_code or 0),
"log_source": log_source,
"log_paths": fallback_paths,
"log_output": trim_output(log_output, 20000),
}}
return True, "diagnostics collected", diagnostics
return False, f"unsupported action: {{action}}", {{"action": action}}
def main():
try:
ok, message, result = execute(ACTION, PAYLOAD)
except Exception as exc:
ok, message, result = False, str(exc), {{"exception": str(exc), "action": ACTION}}
print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
raise SystemExit(0 if ok else 1)
if __name__ == "__main__":
main()
"""
def _extract_json_from_output(text: str) -> tuple[bool, dict]:
for raw_line in reversed(str(text or "").splitlines()):
line = str(raw_line or "").strip()
if not line:
continue
try:
data = json.loads(line)
except Exception:
continue
if isinstance(data, dict) and "ok" in data:
return True, data
return False, {}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,920 @@
from __future__ import annotations
from copy import deepcopy
from app.core.config import settings
from app.services.ops_execution_mode_service import decorate_execution_mode_fields, execution_mode_label
def _split_text_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
text = str(raw_value or "").replace("\r", "\n")
if not text.strip():
return []
normalized = text.replace(",", "\n")
return [item.strip() for item in normalized.split("\n") if item.strip()]
def _coerce_bool(raw_value: object) -> bool:
if isinstance(raw_value, bool):
return raw_value
if isinstance(raw_value, (int, float)):
return bool(raw_value)
normalized = str(raw_value or "").strip().lower()
if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
return True
if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
return False
return bool(normalized)
def _is_missing_value(field_type: str, value: object) -> bool:
if field_type == "text_list":
return not list(value or [])
if field_type == "number":
return value is None
if field_type == "boolean":
return value is None
return str(value or "").strip() == ""
def _service_options() -> list[dict]:
return [
{"label": f"API ({settings.api_service_name})", "value": settings.api_service_name},
{"label": f"Worker ({settings.worker_service_name})", "value": settings.worker_service_name},
{"label": f"Sync Agent ({settings.sync_agent_service_name})", "value": settings.sync_agent_service_name},
{"label": "Node Agent (domaincheck-node-agent)", "value": "domaincheck-node-agent"},
]
def _template_catalog() -> list[dict]:
service_options = _service_options()
return [
{
"group_key": "runtime",
"group_title": "运行时控制",
"group_description": "适合日常按钮化控制 Worker / API / Sync Agent。",
"items": [
{
"key": "runtime.start_worker",
"title": "启动 Worker",
"action": "runtime.start_worker",
"description": "启动目标节点的检测 Worker。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.stop_worker",
"title": "停止 Worker",
"action": "runtime.stop_worker",
"description": "停止目标节点的检测 Worker。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.restart_api",
"title": "重启 API",
"action": "runtime.restart_api",
"description": "重启目标节点的 domaincheck-api 服务。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.start_sync_agent",
"title": "启动 Sync Agent",
"action": "runtime.start_sync_agent",
"description": "启动目标节点的同步代理服务。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "runtime.stop_sync_agent",
"title": "停止 Sync Agent",
"action": "runtime.stop_sync_agent",
"description": "停止目标节点的同步代理服务。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [],
},
{
"key": "service.restart",
"title": "重启任意服务",
"action": "service.restart",
"description": "对指定 systemd 服务执行 restart。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "默认提供 API / Worker / Sync Agent / Node Agent。",
}
],
},
],
},
{
"group_key": "diagnostics",
"group_title": "巡检与取证",
"group_description": "适合做联调、排障、日志回收和节点健康诊断。",
"items": [
{
"key": "health.snapshot",
"title": "采集健康快照",
"action": "health.snapshot",
"description": "采集目标节点关键 systemd 服务状态。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [],
},
{
"key": "logs.collect",
"title": "收集服务日志",
"action": "logs.collect",
"description": "抓取指定服务最近 journalctl 日志。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "默认抓 Worker排 API 问题时可切到 domaincheck-api。",
},
{
"key": "lines",
"label": "日志行数",
"type": "number",
"required": True,
"default": 120,
"min": 20,
"max": 500,
"help": "journalctl -n 的行数,适合短时排障。",
},
],
},
{
"key": "diagnostics.collect",
"title": "收集诊断包",
"action": "diagnostics.collect",
"description": "打包 API / Worker / Sync Agent / Node Agent 的状态与近期日志。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "lines",
"label": "每个服务日志行数",
"type": "number",
"required": True,
"default": 200,
"min": 20,
"max": 800,
"help": "越大越适合深度排障,但任务返回会更重。",
}
],
},
{
"key": "service.status",
"title": "采集服务状态",
"action": "service.status",
"description": "执行 systemctl status --no-pager -l。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "service_name",
"label": "服务名",
"type": "select",
"required": True,
"default": settings.worker_service_name,
"options": service_options,
"help": "用于查看特定服务当前状态和最近日志片段。",
}
],
},
],
},
{
"group_key": "delivery_queue",
"group_title": "回执队列治理",
"group_description": "适合统一处理 Node Agent 的 pending / dead-letter 回执队列,而不是手工登机删文件。",
"items": [
{
"key": "delivery.queue.flush",
"title": "立即冲刷回执队列",
"action": "delivery.queue.flush",
"description": "立即触发目标节点 Node Agent 冲刷 pending 回执队列。",
"risk_level": "low",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": True,
"fields": [
{
"key": "limit",
"label": "本次冲刷上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "限制本次最多冲刷的回执数量,避免一次返回过大。",
}
],
},
{
"key": "delivery.queue.replay",
"title": "重放死信回执",
"action": "delivery.queue.replay",
"description": "把 dead-letter 记录重新放回 pending并可立即触发冲刷。",
"risk_level": "medium",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "record_id",
"label": "单条记录 ID",
"type": "text",
"required": False,
"default": "",
"help": "留空表示按下面的 selector 条件批量匹配。",
},
{
"key": "request_kind",
"label": "记录类型",
"type": "text",
"required": False,
"default": "",
"help": "可选 job_complete / job_event。",
},
{
"key": "detail_code",
"label": "错误码",
"type": "text",
"required": False,
"default": "",
"help": "按 last_detail_code 过滤死信记录。",
},
{
"key": "limit",
"label": "重放上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "批量重放时最多处理多少条死信。",
},
{
"key": "flush_after_replay",
"label": "重放后立即冲刷",
"type": "boolean",
"required": True,
"default": True,
"help": "开启后会把重放回 pending 的记录立即补发一次。",
},
{
"key": "reason",
"label": "重放原因",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可选,便于后续回溯为什么执行此次重放。",
},
],
},
{
"key": "delivery.queue.discard",
"title": "丢弃死信回执",
"action": "delivery.queue.discard",
"description": "把死信移出活动队列,保存到 discarded 归档目录。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "record_id",
"label": "单条记录 ID",
"type": "text",
"required": False,
"default": "",
"help": "留空表示按下面的 selector 条件批量匹配。",
},
{
"key": "request_kind",
"label": "记录类型",
"type": "text",
"required": False,
"default": "",
"help": "可选 job_complete / job_event。",
},
{
"key": "detail_code",
"label": "错误码",
"type": "text",
"required": False,
"default": "",
"help": "按 last_detail_code 过滤死信记录。",
},
{
"key": "limit",
"label": "丢弃上限",
"type": "number",
"required": True,
"default": 20,
"min": 1,
"max": 200,
"help": "批量丢弃时最多处理多少条死信。",
},
{
"key": "reason",
"label": "丢弃原因",
"type": "text",
"required": True,
"default": "",
"wide": True,
"help": "必须说明为何确认这些死信可以被放弃。",
},
],
},
],
},
{
"group_key": "onboarding",
"group_title": "接管与纳管",
"group_description": "适合在控制面生成节点接入工单、统一沉淀首轮接管动作。",
"items": [
{
"key": "node.bootstrap",
"title": "生成节点接入工单",
"action": "node.bootstrap",
"description": "在控制面签发 Node Agent Token并生成 bootstrap env / 脚本 / 一键落地命令。",
"risk_level": "critical",
"default_execution_mode": "control-plane",
"execution_modes": ["control-plane"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "control_plane_base_url",
"label": "控制面地址",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空。留空时由控制面自动回退到当前 API 地址。",
},
{
"key": "root_dir",
"label": "目标安装目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "bootstrap_node_agent.sh 安装根目录。",
},
{
"key": "expires_in_hours",
"label": "Token 有效期(小时)",
"type": "number",
"required": True,
"default": 72,
"min": 1,
"max": 720,
"help": "控制新签发 token 的过期时间。",
},
{
"key": "node_region",
"label": "节点地域提示",
"type": "text",
"required": False,
"default": "",
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
},
{
"key": "node_role",
"label": "节点角色提示",
"type": "text",
"required": False,
"default": "",
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
},
],
},
],
},
{
"group_key": "release",
"group_title": "发布与变更",
"group_description": "适合把单节点灰度、点状修复和手工发布收编到标准 deploy.release 任务。",
"items": [
{
"key": "deploy.release.control",
"title": "发布控制面节点",
"action": "deploy.release",
"description": "面向 controller 节点的标准发布任务,默认会重启 API / Worker / Sync Agent 并执行 API 健康检查。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["control"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [
settings.api_service_name,
settings.worker_service_name,
settings.sync_agent_service_name,
],
"wide": True,
"rows": 3,
"help": "每行一个服务。控制面默认重启 API / Worker / Sync Agent。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": ["http://127.0.0.1:8100/health"],
"wide": True,
"rows": 2,
"help": "每行一个 URL。控制面默认探活本机 API /health。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [
settings.api_service_name,
settings.worker_service_name,
settings.sync_agent_service_name,
],
"wide": True,
"rows": 3,
"help": "每行一个 systemd 服务,发布后会检查其 active 状态。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
{
"key": "deploy.release.worker",
"title": "发布 Worker 节点",
"action": "deploy.release",
"description": "面向独立 Worker 节点的标准发布任务,默认只重启 Worker 并校验 Worker 服务状态。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["worker"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [settings.worker_service_name],
"wide": True,
"rows": 2,
"help": "每行一个服务。独立 Worker 默认只重启 Worker。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 2,
"help": "Worker 节点通常可留空;为空时不做 URL 探活。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [settings.worker_service_name],
"wide": True,
"rows": 2,
"help": "每行一个 systemd 服务,默认只校验 Worker active 状态。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
{
"key": "deploy.release.custom",
"title": "发布自定义节点",
"action": "deploy.release",
"description": "完全自定义 deploy.release 参数,适合灰度验证、特殊节点或后续扩展场景。",
"risk_level": "high",
"default_execution_mode": "remote-agent",
"execution_modes": ["remote-agent", "ssh"],
"target_roles": ["control", "worker"],
"default_auto_approve": False,
"fields": [
{
"key": "release_version",
"label": "版本号",
"type": "text",
"required": True,
"default": "",
"help": "例如 2026.04.18-rc1用于 releases/<version> 目录名。",
},
{
"key": "artifact_url",
"label": "发布包地址",
"type": "textarea",
"required": True,
"default": "",
"wide": True,
"rows": 2,
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
},
{
"key": "checksum",
"label": "SHA256 校验",
"type": "text",
"required": False,
"default": "",
"wide": True,
"help": "可留空;填写后会强校验发布包完整性。",
},
{
"key": "install_root",
"label": "安装根目录",
"type": "text",
"required": True,
"default": "/opt/domaincheck",
"help": "目标节点上的 releases/current 根目录。",
},
{
"key": "restart_services",
"label": "重启服务",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 3,
"help": "每行一个服务。留空表示只落盘版本,不主动重启任何服务。",
},
{
"key": "health_check_urls",
"label": "健康检查 URL",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 2,
"help": "每行一个 URL。留空表示不做 URL 探活。",
},
{
"key": "health_check_services",
"label": "健康检查服务",
"type": "text_list",
"required": False,
"default": [],
"wide": True,
"rows": 3,
"help": "每行一个 systemd 服务。显式留空时不会自动回落到默认 API 检查。",
},
{
"key": "health_check_timeout_seconds",
"label": "URL 超时",
"type": "number",
"required": True,
"default": 10,
"min": 2,
"max": 120,
"help": "每次 URL 探活超时秒数。",
},
{
"key": "health_check_retries",
"label": "重试次数",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 10,
"help": "健康检查失败后的额外重试次数。",
},
{
"key": "health_check_interval_seconds",
"label": "重试间隔",
"type": "number",
"required": True,
"default": 2,
"min": 0,
"max": 60,
"help": "每次重试前的等待秒数。",
},
{
"key": "rollback_on_failure",
"label": "失败自动回滚",
"type": "boolean",
"required": True,
"default": True,
"help": "健康检查失败时自动切回旧 current 并重启服务。",
},
{
"key": "switch_current",
"label": "切换 current 软链",
"type": "boolean",
"required": True,
"default": True,
"help": "关闭后只解压版本目录,不切 current。",
},
],
},
],
},
]
def get_ops_action_templates() -> dict:
groups = deepcopy(_template_catalog())
items: list[dict] = []
for group in groups:
group_key = str(group.get("group_key") or "").strip()
group_title = str(group.get("group_title") or "").strip()
group_description = str(group.get("group_description") or "").strip()
normalized_group_items: list[dict] = []
for item in list(group.get("items") or []):
normalized_item = decorate_execution_mode_fields(dict(item))
normalized_item["group_key"] = group_key
normalized_item["group_title"] = group_title
normalized_item["group_description"] = group_description
normalized_group_items.append(normalized_item)
items.append(normalized_item)
group["items"] = normalized_group_items
return {
"groups": groups,
"items": items,
"defaults": {
"requested_by": "web-ui",
"execution_mode": "remote-agent",
"execution_mode_label": execution_mode_label("remote-agent"),
},
}
def get_ops_action_template(template_key: str) -> dict:
normalized_key = str(template_key or "").strip()
if not normalized_key:
return {}
for group in _template_catalog():
group_key = str(group.get("group_key") or "").strip()
group_title = str(group.get("group_title") or "").strip()
group_description = str(group.get("group_description") or "").strip()
for item in list(group.get("items") or []):
if str(item.get("key") or "").strip() == normalized_key:
normalized_item = decorate_execution_mode_fields(deepcopy(item))
normalized_item["group_key"] = group_key
normalized_item["group_title"] = group_title
normalized_item["group_description"] = group_description
return normalized_item
return {}
def build_ops_template_payload(template_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
template = get_ops_action_template(template_key)
if not template:
return False, "动作模板不存在", {}
source_payload = dict(payload or {})
normalized_payload: dict = {}
for field in list(template.get("fields") or []):
field_key = str(field.get("key") or "").strip()
if not field_key:
continue
field_type = str(field.get("type") or "text").strip()
raw_value = source_payload.get(field_key, field.get("default"))
if field_type == "number":
try:
value = int(raw_value or 0)
except Exception:
return False, f"{field_key} 必须是数字", {}
min_value = field.get("min")
max_value = field.get("max")
if min_value is not None:
value = max(int(min_value), value)
if max_value is not None:
value = min(int(max_value), value)
elif field_type == "boolean":
value = _coerce_bool(raw_value)
elif field_type == "text_list":
value = _split_text_list(raw_value)
else:
value = str(raw_value or "").strip()
if bool(field.get("required", False)) and _is_missing_value(field_type, value):
return False, f"{field_key} 不能为空", {}
normalized_payload[field_key] = value
return True, "ok", normalized_payload

View File

@@ -7,6 +7,7 @@ from app.core.config import settings
from app.core.db import get_db
from app.core.files import read_json
from app.core.redis_client import get_redis
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
@@ -187,6 +188,89 @@ def _build_multi_region_readiness(
}
def _detect_participation_snapshot(*, row: dict) -> dict:
items_running = int(row.get("items_running", 0) or 0)
items_claimed = int(row.get("items_claimed", 0) or 0)
processed_recent = int(row.get("processed_recent", 0) or 0)
current_load = int(row.get("current_load", 0) or 0)
status = str(row.get("status") or "").strip().lower()
if items_running > 0:
return {
"participation_state": "running",
"participation_label": "执行中",
"participation_reason": f"当前正在执行 {items_running} 项检测任务。",
"is_current_participant": True,
"is_dispatch_active": True,
}
if items_claimed > 0:
return {
"participation_state": "claimed",
"participation_label": "已领待跑",
"participation_reason": f"已领取 {items_claimed} 项任务,等待线程继续执行。",
"is_current_participant": True,
"is_dispatch_active": True,
}
if processed_recent > 0:
return {
"participation_state": "recent_throughput",
"participation_label": "近窗有吞吐",
"participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。",
"is_current_participant": True,
"is_dispatch_active": False,
}
if current_load > 0 or status == "busy":
load_value = max(current_load, 1)
return {
"participation_state": "load_syncing",
"participation_label": "负载待确认",
"participation_reason": f"节点当前负载为 {load_value},但还未观察到已领、执行中或近窗吞吐数据,先归入在线未参与观察。",
"is_current_participant": False,
"is_dispatch_active": False,
}
return {
"participation_state": "standby",
"participation_label": "在线待命",
"participation_reason": "当前未领任务、未执行任务,也没有近窗吞吐。",
"is_current_participant": False,
"is_dispatch_active": False,
}
def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict:
metadata = cluster_node.get("metadata") or {}
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
region = str(cluster_node.get("region") or settings.node_region)
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
row = {
"node_code": node_code,
"role": role,
"region": region,
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": is_effective_worker,
"detect_participating": False,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": items_total,
"items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
"items_claimed": items_claimed,
"items_running": items_running,
"items_completed": items_completed,
"items_failed": items_failed,
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
"metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"),
}
row.update(_detect_participation_snapshot(row=row))
row["detect_participating"] = bool(row.get("is_current_participant", False))
return row
def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
active_job = detect_snapshot.get("active_job") or {}
@@ -204,56 +288,25 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
}
merged: list[dict] = []
seen: set[str] = set()
for item in list(active_job.get("node_stats") or []):
node_code = str(item.get("node_code") or "").strip()
if not node_code or node_code == "unassigned":
continue
seen.add(node_code)
job_map = {
str(item.get("node_code") or "").strip(): item
for item in list(active_job.get("node_stats") or [])
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
}
candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map))
for node_code in candidate_codes:
cluster_node = cluster_map.get(node_code, {})
queue_node = queue_map.get(node_code, {})
merged.append(
{
"node_code": node_code,
"role": str(cluster_node.get("role") or "worker"),
"region": str(cluster_node.get("region") or settings.node_region),
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": bool(cluster_node.get("is_effective_worker", False) or str(cluster_node.get("role") or "") == "worker"),
"detect_participating": True,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": int(item.get("items_total", 0) or 0),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
}
if cluster_node and not bool(cluster_node.get("is_effective_worker", False)):
continue
row = _build_detect_node_row(
node_code=node_code,
cluster_node=cluster_node,
job_node=job_map.get(node_code, {}),
queue_node=queue_map.get(node_code, {}),
)
if worker_runtime.get("running", False) and settings.node_code not in seen:
cluster_node = cluster_map.get(settings.node_code, {})
if cluster_node:
merged.append(
{
"node_code": settings.node_code,
"role": str(cluster_node.get("role") or settings.node_role),
"region": str(cluster_node.get("region") or settings.node_region),
"status": str(cluster_node.get("status") or "online"),
"is_effective_worker": True,
"detect_participating": True,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_failed": 0,
"processed_recent": 0,
"processed_per_minute": 0,
}
)
if not bool(row.get("is_current_participant", False)):
continue
merged.append(row)
return sorted(
merged,
@@ -261,11 +314,143 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
-int(item.get("items_running", 0) or 0),
-int(item.get("items_claimed", 0) or 0),
-int(item.get("processed_recent", 0) or 0),
-int(item.get("current_load", 0) or 0),
str(item.get("node_code") or ""),
),
)
def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
participating_codes = {
str(item.get("node_code") or "").strip()
for item in list(participating_nodes or [])
if str(item.get("node_code") or "").strip()
}
standby_rows: list[dict] = []
for node in cluster_nodes:
node_code = str(node.get("node_code") or "").strip()
node_status = str(node.get("status") or "").strip()
node_current_load = int(node.get("current_load", 0) or 0)
metadata = node.get("metadata") or {}
if not node_code:
continue
if not bool(node.get("is_effective_worker", False)):
continue
if node_status not in {"online", "busy"}:
continue
if node_code in participating_codes:
continue
standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby"
standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命"
standby_reason = (
str(metadata.get("detail") or "").strip()
or str(metadata.get("phase_detail") or "").strip()
or (
f"当前阶段:{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}"
if str(metadata.get("phase") or metadata.get("phase_label") or "").strip()
else ""
)
or (
f"节点当前负载为 {node_current_load},但还未观察到已领、执行中或近窗吞吐数据。"
if standby_state == "load_syncing"
else "节点在线,当前未领任务、未执行任务,也没有近窗吞吐。"
)
)
standby_rows.append(
{
"node_code": node_code,
"role": str(node.get("role") or "worker"),
"region": str(node.get("region") or settings.node_region),
"status": node_status,
"is_effective_worker": True,
"detect_participating": False,
"participation_state": standby_state,
"participation_label": standby_label,
"participation_reason": standby_reason,
"current_load": node_current_load,
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
"phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
"detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
"worker_online": bool(metadata.get("worker_online", False)),
"standby_reason": standby_reason,
}
)
return sorted(
standby_rows,
key=lambda item: (
str(item.get("status") or ""),
str(item.get("role") or ""),
str(item.get("node_code") or ""),
),
)
def _build_detect_participation_summary(
*,
participating_nodes: list[dict],
standby_nodes: list[dict],
cluster_snapshot: dict,
) -> dict:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
effective_online_nodes = [
node
for node in cluster_nodes
if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"}
]
dispatch_active_nodes = [
row for row in participating_nodes
if bool(row.get("is_dispatch_active", False))
]
recent_only_nodes = [
row for row in participating_nodes
if str(row.get("participation_state") or "").strip() == "recent_throughput"
]
load_syncing_nodes = [
row for row in standby_nodes
if str(row.get("participation_state") or "").strip() == "load_syncing"
]
pure_standby_nodes = [
row for row in standby_nodes
if str(row.get("participation_state") or "").strip() == "standby"
]
dedicated_worker_nodes = [
node for node in effective_online_nodes
if str(node.get("role") or "").strip() == "worker"
]
controller_worker_nodes = [
node for node in effective_online_nodes
if str(node.get("role") or "").strip() == "control"
]
summary_parts = [
f"有效执行节点 {len(effective_online_nodes)}",
f"正在执行/领任务 {len(dispatch_active_nodes)}",
f"近窗刚有吞吐 {len(recent_only_nodes)}",
f"在线但未参与 {len(standby_nodes)}",
]
if load_syncing_nodes:
summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)}")
return {
"effective_online_nodes": len(effective_online_nodes),
"participating_nodes": len(participating_nodes),
"dispatch_active_nodes": len(dispatch_active_nodes),
"recent_only_nodes": len(recent_only_nodes),
"non_participating_nodes": len(standby_nodes),
"standby_nodes": len(pure_standby_nodes),
"load_syncing_nodes": len(load_syncing_nodes),
"dedicated_worker_nodes": len(dedicated_worker_nodes),
"controller_worker_nodes": len(controller_worker_nodes),
"dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes],
"recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes],
"non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes],
"standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes],
"load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes],
"summary": "".join(summary_parts),
}
def get_runtime_status() -> dict:
runtime_settings = get_runtime_settings()
worker_runtime = detect_worker_runtime()
@@ -314,6 +499,17 @@ def get_runtime_status() -> dict:
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"queue_health": queue_health,
"capacity_plan": capacity_plan,
"log_sync": {
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
"mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"),
"line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0),
"source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0),
"source_nodes": list(detect_snapshot.get("remote_log_nodes") or []),
"source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []),
"last_at": str(detect_snapshot.get("remote_log_last_at") or ""),
"last_line": str(detect_snapshot.get("remote_log_last_line") or ""),
"preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:],
},
}
if not worker_expected_on_this_node:
detect_payload.update(
@@ -349,6 +545,16 @@ def get_runtime_status() -> dict:
detect_snapshot=detect_payload,
worker_runtime=worker_runtime,
)
detect_payload["standby_nodes"] = _build_standby_detect_nodes(
cluster_snapshot=cluster_snapshot,
participating_nodes=detect_payload["participating_nodes"],
)
detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or [])
detect_payload["participation_summary"] = _build_detect_participation_summary(
participating_nodes=detect_payload["participating_nodes"],
standby_nodes=detect_payload["non_participating_nodes"],
cluster_snapshot=cluster_snapshot,
)
append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
sync_summary = get_sync_summary(record_limit=5)
readiness = _build_multi_region_readiness(
@@ -357,6 +563,7 @@ def get_runtime_status() -> dict:
worker_runtime=worker_runtime,
sync_agent_runtime=sync_agent_runtime,
)
build_info = get_runtime_build_info()
return {
"api": {
@@ -371,6 +578,7 @@ def get_runtime_status() -> dict:
"health_url": f"http://127.0.0.1:{settings.api_port}/health",
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
"build": build_info,
},
"node": {
"code": settings.node_code,

View File

@@ -450,6 +450,17 @@ def append_runtime_projection_if_changed(
continue
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
break
local_job_bucket = {}
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
continue
local_job_bucket = item
break
if not local_participating:
local_participating = bool(
int(local_job_bucket.get("items_running", 0) or 0) > 0
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
)
projection = {
"node": {
"node_code": settings.node_code,

File diff suppressed because it is too large Load Diff

View File

@@ -12,7 +12,7 @@ TEMPLATE_DIR="$API_DIR/deploy/multi-region/templates"
if [ "$ROLE" != "controller" ] && [ "$ROLE" != "worker" ]; then
echo "invalid role: $ROLE"
echo "usage: bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]"
echo "usage: bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]"
exit 1
fi

View File

@@ -0,0 +1,61 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="${1:-/opt/domaincheck}"
ENV_FILE="/etc/default/domaincheck-node-agent"
SERVICE_FILE="/etc/systemd/system/domaincheck-node-agent.service"
SERVICE_NAME="domaincheck-node-agent"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
resolve_project_root() {
local candidate
for candidate in \
"${ROOT_DIR}/domainCheck" \
"${ROOT_DIR}/domain-api" \
"${ROOT_DIR}" \
"$(cd "${SCRIPT_DIR}/../.." && pwd)"; do
if [[ -f "${candidate}/deploy/systemd/domain-node-agent.service" && -f "${candidate}/deploy/multi-region/templates/domaincheck-node-agent.env.example" ]]; then
printf '%s' "${candidate}"
return 0
fi
done
return 1
}
PROJECT_ROOT="$(resolve_project_root || true)"
if [[ -z "${PROJECT_ROOT}" ]]; then
echo "unable to resolve project root under ${ROOT_DIR} or ${SCRIPT_DIR}" >&2
exit 1
fi
TEMPLATE_FILE="${PROJECT_ROOT}/deploy/multi-region/templates/domaincheck-node-agent.env.example"
UNIT_TEMPLATE="${PROJECT_ROOT}/deploy/systemd/domain-node-agent.service"
echo "[1/5] install systemd unit"
install -m 0644 "${UNIT_TEMPLATE}" "${SERVICE_FILE}"
echo "[2/5] install env template if absent"
if [[ ! -f "${ENV_FILE}" ]]; then
install -m 0644 "${TEMPLATE_FILE}" "${ENV_FILE}"
echo "created ${ENV_FILE}"
else
echo "keep existing ${ENV_FILE}"
fi
echo "[3/5] reload systemd"
systemctl daemon-reload
echo "[4/5] enable service"
systemctl enable "${SERVICE_NAME}" >/dev/null
echo "[5/5] next steps"
echo "project_root=${PROJECT_ROOT}"
echo "edit ${ENV_FILE} and set:"
echo " OPS_CONTROL_PLANE_BASE_URL"
echo " OPS_AGENT_TOKEN"
echo " NODE_CODE / NODE_REGION / NODE_ROLE"
echo
echo "then run:"
echo " systemctl restart ${SERVICE_NAME}"
echo " systemctl status ${SERVICE_NAME} --no-pager -l"

View File

@@ -0,0 +1,91 @@
#!/usr/bin/env bash
set -euo pipefail
API_BASE_URL="${1:-http://127.0.0.1:8100}"
NODE_CODE="${2:-}"
NODE_REGION="${3:-mainland}"
NODE_ROLE="${4:-worker}"
CONTROL_PLANE_BASE_URL="${5:-${API_BASE_URL}}"
ROOT_DIR="${6:-/opt/domaincheck}"
ISSUED_BY="${ISSUED_BY:-cli-bootstrap}"
EXPIRES_IN_HOURS="${EXPIRES_IN_HOURS:-72}"
if [[ -z "${NODE_CODE}" ]]; then
echo "usage: $0 <api_base_url> <node_code> [node_region] [node_role] [control_plane_base_url] [root_dir]" >&2
exit 1
fi
NORMALIZED_API_BASE_URL="${API_BASE_URL%/}"
NORMALIZED_CONTROL_PLANE_BASE_URL="${CONTROL_PLANE_BASE_URL%/}"
if [[ "${NORMALIZED_API_BASE_URL}" != */api/v1 ]]; then
API_ENDPOINT="${NORMALIZED_API_BASE_URL}/api/v1/ops/agent/bootstrap-plan"
else
API_ENDPOINT="${NORMALIZED_API_BASE_URL}/ops/agent/bootstrap-plan"
fi
TMP_RESPONSE_FILE="$(mktemp)"
cleanup() {
rm -f "${TMP_RESPONSE_FILE}"
}
trap cleanup EXIT
python3 - "${NODE_CODE}" "${NODE_REGION}" "${NODE_ROLE}" "${ISSUED_BY}" "${EXPIRES_IN_HOURS}" "${NORMALIZED_CONTROL_PLANE_BASE_URL}" "${ROOT_DIR}" >"${TMP_RESPONSE_FILE}.payload" <<'PY'
import json
import sys
payload = {
"node_code": sys.argv[1],
"node_region": sys.argv[2],
"node_role": sys.argv[3],
"issued_by": sys.argv[4],
"expires_in_hours": int(sys.argv[5]),
"control_plane_base_url": sys.argv[6],
"root_dir": sys.argv[7],
"metadata": {
"issued_from": "build_node_agent_bootstrap_plan.sh",
},
}
print(json.dumps(payload, ensure_ascii=False))
PY
curl -fsS -X POST \
-H 'Content-Type: application/json' \
--data-binary @"${TMP_RESPONSE_FILE}.payload" \
"${API_ENDPOINT}" >"${TMP_RESPONSE_FILE}"
python3 - "${TMP_RESPONSE_FILE}" <<'PY'
import json
import sys
from pathlib import Path
response = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
response_code = response.get("code", 0)
if int(response_code if response_code is not None else 0) != 0:
raise SystemExit(response.get("message") or "bootstrap plan request failed")
data = response.get("data") or {}
plan = data.get("bootstrap_plan") or {}
print("[token]")
print(f"node_code={data.get('node_code', '')}")
print(f"token_preview={data.get('token_preview', '')}")
print(f"expires_at={data.get('expires_at', '')}")
print()
print("[env]")
print(plan.get("env_content") or "")
print()
print("[commands]")
print(plan.get("command_block") or "")
print()
print("[bootstrap-script]")
print(plan.get("bootstrap_script_content") or "")
print()
print("[bootstrap-run-block]")
print(plan.get("bootstrap_run_script_block") or "")
print()
print("[health-checks]")
for item in plan.get("health_checks") or []:
print(item)
PY
rm -f "${TMP_RESPONSE_FILE}.payload"

View File

@@ -0,0 +1,345 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
API_SERVICE="${API_SERVICE:-domaincheck-api}"
WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}"
NODE_AGENT_SERVICE="${NODE_AGENT_SERVICE:-domaincheck-node-agent}"
API_ENV_FILE="${API_ENV_FILE:-/etc/default/domaincheck-api}"
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/default/domaincheck-worker}"
NODE_AGENT_ENV_FILE="${NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
API_DIR="${PROJECT_ROOT}/domain-api"
WEB_DIR="${PROJECT_ROOT}/domain-web"
PYTHON_BIN="${PYTHON_BIN:-python3}"
FETCH_BODY=""
FETCH_STATUS="000"
fetch_json() {
local url="${1:-}"
local raw_text
raw_text="$(curl -sS \
--connect-timeout "${CURL_CONNECT_TIMEOUT:-3}" \
--max-time "${CURL_MAX_TIME:-10}" \
"${url}" \
-w $'\nHTTP_STATUS=%{http_code}' || true)"
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
}
echo "[1/8] local command availability"
for cmd in bash curl systemctl "${PYTHON_BIN}" node npm; do
if command -v "${cmd}" >/dev/null 2>&1; then
printf '%s: ok -> %s\n' "${cmd}" "$(command -v "${cmd}")"
else
printf '%s: missing\n' "${cmd}"
fi
done
echo
echo "[2/8] python environment"
"${PYTHON_BIN}" - <<'PY' "${API_DIR}" "${WEB_DIR}"
import importlib.util
import json
import pathlib
import sys
api_dir = pathlib.Path(sys.argv[1])
web_dir = pathlib.Path(sys.argv[2])
def has_module(name: str) -> bool:
return importlib.util.find_spec(name) is not None
payload = {
"python_executable": sys.executable,
"python_version": sys.version.split()[0],
"pytest_installed": has_module("pytest"),
"uvicorn_installed": has_module("uvicorn"),
"fastapi_installed": has_module("fastapi"),
"api_requirements_exists": (api_dir / "requirements.txt").exists(),
"web_package_json_exists": (web_dir / "package.json").exists(),
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY
echo
echo "[3/8] env files"
for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${NODE_AGENT_ENV_FILE}"; do
if [[ -f "${env_file}" ]]; then
echo "exists: ${env_file}"
else
echo "missing: ${env_file}"
fi
done
echo
echo "[4/8] systemd service states"
for service in "${API_SERVICE}" "${WORKER_SERVICE}" "${SYNC_AGENT_SERVICE}" "${NODE_AGENT_SERVICE}"; do
if systemctl list-unit-files "${service}.service" --no-pager >/dev/null 2>&1; then
printf '%s: enabled=%s active=%s\n' \
"${service}" \
"$(systemctl is-enabled "${service}" 2>/dev/null || true)" \
"$(systemctl is-active "${service}" 2>/dev/null || true)"
else
printf '%s: not-installed\n' "${service}"
fi
done
echo
echo "[5/8] runtime preflight"
fetch_json "${BASE_URL}/api/v1/runtime/preflight"
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
echo "[6/8] runtime readiness"
fetch_json "${BASE_URL}/api/v1/runtime/readiness"
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
echo "[7/8] runtime build-info"
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
echo "[8/8] condensed env audit summary"
"${PYTHON_BIN}" - <<'PY' \
"${BASE_URL}" \
"${API_SERVICE}" \
"${WORKER_SERVICE}" \
"${SYNC_AGENT_SERVICE}" \
"${NODE_AGENT_SERVICE}" \
"${API_ENV_FILE}" \
"${WORKER_ENV_FILE}" \
"${NODE_AGENT_ENV_FILE}" \
"${PROJECT_ROOT}"
import importlib.util
import json
import pathlib
import subprocess
import sys
import urllib.request
from typing import Dict, List
base_url, api_service, worker_service, sync_agent_service, node_agent_service, api_env, worker_env, node_agent_env, project_root = sys.argv[1:10]
project_root_path = pathlib.Path(project_root)
def has_module(name: str) -> bool:
return importlib.util.find_spec(name) is not None
def inspect_python_runtime(path: str) -> Dict[str, object]:
runtime_path = pathlib.Path(path)
if not runtime_path.exists():
return {}
probe = r"""
import importlib.util
import json
import sys
mods = ["pytest", "fastapi", "uvicorn"]
print(json.dumps({
"python_executable": sys.executable,
"python_version": sys.version.split()[0],
"modules": {name: importlib.util.find_spec(name) is not None for name in mods},
}, ensure_ascii=False))
"""
try:
raw = subprocess.check_output([str(runtime_path), "-c", probe], text=True, stderr=subprocess.DEVNULL)
except Exception:
return {}
try:
payload = json.loads(raw)
except Exception:
return {}
payload["path"] = str(runtime_path)
return payload
def systemd_state(service: str) -> dict:
try:
enabled = subprocess.check_output(
["systemctl", "is-enabled", service],
text=True,
stderr=subprocess.DEVNULL,
).strip()
except Exception:
enabled = ""
try:
active = subprocess.check_output(
["systemctl", "is-active", service],
text=True,
stderr=subprocess.DEVNULL,
).strip()
except Exception:
active = ""
return {"enabled": enabled, "active": active}
def fetch_json(url: str) -> tuple[str, dict]:
try:
with urllib.request.urlopen(url, timeout=10) as resp:
status = str(resp.status)
raw = resp.read().decode("utf-8", errors="replace")
except Exception:
return "000", {}
try:
payload = json.loads(raw)
except Exception:
return status, {}
return status, payload.get("data") if isinstance(payload, dict) else {}
def read_git_value(*args: str) -> str:
try:
result = subprocess.check_output(
["git", "-C", str(project_root_path), *args],
text=True,
stderr=subprocess.DEVNULL,
).strip()
except Exception:
return ""
return result
def read_text(path: pathlib.Path) -> str:
try:
return path.read_text(encoding="utf-8", errors="replace")
except Exception:
return ""
local_commit_sha = read_git_value("rev-parse", "--short=12", "HEAD")
local_commit_ref = read_git_value("rev-parse", "--abbrev-ref", "HEAD")
ops_agent_service_text = read_text(project_root_path / "domain-api" / "app" / "services" / "ops_agent_service.py")
bootstrap_node_agent_text = read_text(project_root_path / "domain-api" / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
repo_supports_install_command_block = (
"install_command_block" in ops_agent_service_text
and "_candidate_node_agent_install_paths" in ops_agent_service_text
)
repo_supports_multi_layout_bootstrap = "resolve_project_root" in bootstrap_node_agent_text
preflight_status, preflight = fetch_json(f"{base_url}/api/v1/runtime/preflight")
readiness_status, readiness = fetch_json(f"{base_url}/api/v1/runtime/readiness")
build_status, build = fetch_json(f"{base_url}/api/v1/runtime/build-info")
route_surface = build.get("route_surface") if isinstance(build, dict) else {}
missing = []
tooling_items: List[str] = []
for env_file in [api_env, worker_env, node_agent_env]:
if not pathlib.Path(env_file).exists():
missing.append(f"missing_env:{env_file}")
python_candidates: List[str] = []
seen_candidates = set()
for candidate in [
"/opt/domaincheck/domainCheck/.venv/bin/python",
str(pathlib.Path.cwd() / ".venv" / "bin" / "python"),
str(pathlib.Path.cwd() / "domain-api" / ".venv" / "bin" / "python"),
pathlib.Path(sys.executable).as_posix(),
]:
if candidate and candidate not in seen_candidates:
seen_candidates.add(candidate)
python_candidates.append(candidate)
python_runtimes = [item for item in (inspect_python_runtime(candidate) for candidate in python_candidates) if item]
def runtime_has_module(module_name: str) -> bool:
for runtime in python_runtimes:
modules = runtime.get("modules") or {}
if bool(modules.get(module_name)):
return True
return False
if not runtime_has_module("fastapi"):
missing.append("python_module:fastapi")
if not runtime_has_module("uvicorn"):
missing.append("python_module:uvicorn")
if not runtime_has_module("pytest"):
tooling_items.append("python_module:pytest")
runtime_commit_sha = str(build.get("commit_sha") or "").strip() if isinstance(build, dict) else ""
runtime_commit_ref = str(build.get("commit_ref") or "").strip() if isinstance(build, dict) else ""
runtime_route_missing_keys = [str(item).strip() for item in list((route_surface or {}).get("missing_keys") or []) if str(item).strip()]
runtime_expected_paths = {
str(key).strip(): str(value).strip()
for key, value in dict((route_surface or {}).get("expected_paths") or {}).items()
if str(key).strip()
}
runtime_bootstrap_plan_route_missing = "ops_node_handover_bootstrap_plan" in runtime_route_missing_keys
runtime_declares_bootstrap_plan_route = "ops_node_handover_bootstrap_plan" in runtime_expected_paths
runtime_repo_commit_drift = bool(local_commit_sha and runtime_commit_sha and local_commit_sha != runtime_commit_sha)
runtime_repo_capability_drift = bool(
repo_supports_install_command_block
and (runtime_bootstrap_plan_route_missing or not runtime_declares_bootstrap_plan_route)
)
recommended_actions: List[str] = []
if runtime_repo_capability_drift:
recommended_actions.append("systemctl restart domaincheck-api")
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01")
elif runtime_repo_commit_drift:
recommended_actions.append("systemctl restart domaincheck-api")
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100")
payload = {
"status": "ready",
"headline": "当前环境没有发现新的基础阻断,可继续做上线复核。",
"repo": {
"project_root": str(project_root_path),
"commit_sha": local_commit_sha,
"commit_ref": local_commit_ref,
"supports_install_command_block": repo_supports_install_command_block,
"supports_multi_layout_bootstrap": repo_supports_multi_layout_bootstrap,
},
"python": {
"pytest_installed": runtime_has_module("pytest"),
"fastapi_installed": runtime_has_module("fastapi"),
"uvicorn_installed": runtime_has_module("uvicorn"),
"runtimes": python_runtimes,
},
"services": {
api_service: systemd_state(api_service),
worker_service: systemd_state(worker_service),
sync_agent_service: systemd_state(sync_agent_service),
node_agent_service: systemd_state(node_agent_service),
},
"runtime": {
"preflight_status_code": preflight_status,
"preflight_ok": bool((preflight or {}).get("ok", False)),
"readiness_status_code": readiness_status,
"readiness_status": str((readiness or {}).get("status") or ""),
"readiness_summary": str((readiness or {}).get("summary") or ""),
"build_status_code": build_status,
"build_commit_sha": runtime_commit_sha,
"build_commit_ref": runtime_commit_ref,
"route_surface_complete": bool((route_surface or {}).get("surface_complete", False)),
"route_surface_missing_keys": runtime_route_missing_keys,
"route_surface_declares_bootstrap_plan": runtime_declares_bootstrap_plan_route,
"repo_commit_drift": runtime_repo_commit_drift,
"repo_capability_drift": runtime_repo_capability_drift,
"runtime_may_need_restart": runtime_repo_commit_drift or runtime_repo_capability_drift,
},
"missing_items": missing,
"tooling_items": tooling_items,
"recommended_actions": recommended_actions,
}
if not bool((preflight or {}).get("ok", False)):
payload["status"] = "blocked"
payload["headline"] = "runtime/preflight 未通过,先修环境与依赖,再继续上线复核。"
elif str((readiness or {}).get("status") or "") == "blocking":
payload["status"] = "blocked"
payload["headline"] = "runtime/readiness 当前为 blocking现场仍不适合进入正式发布。"
elif runtime_repo_capability_drift:
payload["status"] = "attention"
payload["headline"] = "本地仓库已经具备新的 Node Agent 接管能力,但运行中的 API 路由面仍旧,当前更像是服务未重启到最新代码。"
elif runtime_repo_commit_drift:
payload["status"] = "attention"
payload["headline"] = "本地仓库 commit 与运行中 API build-info 不一致,建议先重启或重发当前版本,再继续上线复核。"
elif not bool((route_surface or {}).get("surface_complete", False)):
payload["status"] = "attention"
payload["headline"] = "运行中 API 路由面还不完整,建议先确认当前实例是否已经吃到最新版本。"
elif missing:
payload["status"] = "attention"
payload["headline"] = "基础环境存在缺口,虽然未必阻断运行,但建议先补齐后再交付。"
elif tooling_items:
payload["status"] = "ready"
payload["headline"] = "运行环境可继续上线复核,但本机工具链仍有缺口,建议后续补齐。"
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,423 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
OVERSEAS_BASE_URL="${2:-}"
VIEW_MODE="${3:-full}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
LOCAL_NODE_AGENT_SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
LOCAL_NODE_AGENT_ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
FETCH_BODY=""
FETCH_STATUS="000"
fetch_json() {
local url="${1:-}"
local raw_text
raw_text="$(curl -sS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"${url}" \
-w $'\nHTTP_STATUS=%{http_code}' || true)"
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
}
run_local_node_agent_probe() {
SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME}" \
ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE}" \
bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check_node_agent.sh"
}
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}"
STACK_JSON="${FETCH_BODY}"
STACK_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/go-live-summary?base_url=${BASE_URL}"
GO_LIVE_API_JSON="${FETCH_BODY}"
GO_LIVE_API_STATUS="${FETCH_STATUS}"
if [[ "${GO_LIVE_API_STATUS}" == 2* ]]; then
if [[ "${VIEW_MODE}" != "summary" ]]; then
echo "[1/1] go-live summary (api)"
fi
printf '%s\n' "${GO_LIVE_API_JSON}"
exit 0
fi
fetch_json "${BASE_URL}/api/v1/ops/contracts"
CONTRACTS_JSON="${FETCH_BODY}"
CONTRACTS_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad"
LAUNCHPAD_JSON="${FETCH_BODY}"
LAUNCHPAD_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
BUILD_INFO_JSON="${FETCH_BODY}"
BUILD_INFO_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/nodes"
NODES_JSON="${FETCH_BODY}"
NODES_STATUS="${FETCH_STATUS}"
fetch_json "${BASE_URL}/api/v1/ops/overview"
OVERVIEW_JSON="${FETCH_BODY}"
OVERVIEW_STATUS="${FETCH_STATUS}"
LOCAL_NODE_AGENT_RAW="$(run_local_node_agent_probe 2>&1 || true)"
if [[ "${VIEW_MODE}" != "summary" ]]; then
echo "[1/7] ops stack diagnosis"
printf 'status=%s\n%s\n\n' "${STACK_STATUS}" "${STACK_JSON}"
echo "[2/7] ops contracts"
printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}"
echo "[3/7] release launchpad"
printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}"
echo "[4/7] runtime build info"
printf 'status=%s\n%s\n\n' "${BUILD_INFO_STATUS}" "${BUILD_INFO_JSON}"
echo "[5/7] managed nodes"
printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}"
echo "[6/7] ops overview"
printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}"
echo "[7/7] local node agent probe"
printf '%s\n\n' "${LOCAL_NODE_AGENT_RAW}"
fi
echo "[8/8] go-live summary"
python3 - <<'PY' \
"${BASE_URL}" \
"${OVERSEAS_BASE_URL}" \
"${STACK_JSON}" \
"${STACK_STATUS}" \
"${CONTRACTS_JSON}" \
"${CONTRACTS_STATUS}" \
"${LAUNCHPAD_JSON}" \
"${LAUNCHPAD_STATUS}" \
"${BUILD_INFO_JSON}" \
"${BUILD_INFO_STATUS}" \
"${NODES_JSON}" \
"${NODES_STATUS}" \
"${OVERVIEW_JSON}" \
"${OVERVIEW_STATUS}" \
"${LOCAL_NODE_AGENT_RAW}" \
"${LOCAL_NODE_AGENT_SERVICE_NAME}"
import json
import sys
def load_payload(raw_text: str) -> dict:
try:
payload = json.loads(str(raw_text or ""))
except Exception:
return {}
return payload if isinstance(payload, dict) else {}
def load_data(raw_text: str) -> dict:
payload = load_payload(raw_text)
data = payload.get("data")
return data if isinstance(data, dict) else {}
def ensure_dict(value):
return value if isinstance(value, dict) else {}
def ensure_list(value):
return value if isinstance(value, list) else []
def ensure_str(value):
return value if isinstance(value, str) else ""
def truthy_http(status_code: str) -> bool:
return str(status_code or "").startswith("2")
base_url = str(sys.argv[1] or "http://127.0.0.1:8100").strip()
overseas_base_url = str(sys.argv[2] or "").strip()
stack_payload = load_data(sys.argv[3])
stack_status_code = str(sys.argv[4] or "000")
contracts_payload = load_data(sys.argv[5])
contracts_status_code = str(sys.argv[6] or "000")
launchpad_payload = load_data(sys.argv[7])
launchpad_status_code = str(sys.argv[8] or "000")
build_payload = load_data(sys.argv[9])
build_status_code = str(sys.argv[10] or "000")
nodes_payload = load_data(sys.argv[11])
nodes_status_code = str(sys.argv[12] or "000")
overview_payload = load_data(sys.argv[13])
overview_status_code = str(sys.argv[14] or "000")
local_node_agent_raw = str(sys.argv[15] or "")
local_node_agent_service_name = str(sys.argv[16] or "domaincheck-node-agent").strip()
diagnosis = ensure_dict(stack_payload.get("diagnosis"))
release_hub = ensure_dict(stack_payload.get("release_hub"))
managed_nodes = ensure_dict(stack_payload.get("managed_nodes"))
runtime_build_info = ensure_dict(stack_payload.get("runtime_build_info")) or build_payload
launchpad_status = ensure_dict(launchpad_payload.get("launchpad_status"))
launchpad_recommended_target_node_code = ensure_str(launchpad_status.get("recommended_target_node_code"))
launchpad_recommended_recovery_label = ensure_str(launchpad_status.get("recommended_recovery_label"))
launchpad_recommended_recovery_summary = ensure_str(launchpad_status.get("recommended_recovery_summary"))
launchpad_onboarding_bootstrap_pending_nodes = int(
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
)
launchpad_onboarding_acceptance_ready_nodes = int(
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
)
contracts = ensure_list(contracts_payload.get("contracts"))
nodes_summary = ensure_dict(nodes_payload.get("summary")) or ensure_dict(managed_nodes.get("summary"))
route_surface = ensure_dict(runtime_build_info.get("route_surface"))
next_step = ensure_dict(diagnosis.get("next_step"))
operator_decision = ensure_dict(diagnosis.get("operator_decision"))
overview_recommendation = ensure_dict(overview_payload.get("recommendation"))
execution_scene = ensure_dict(overview_payload.get("execution_scene"))
log_sync = ensure_dict(execution_scene.get("log_sync"))
local_agent_enabled = ""
local_agent_active = ""
local_agent_installed = False
for line in [item.strip() for item in local_node_agent_raw.splitlines() if item.strip()]:
if line == "not-installed":
local_agent_installed = False
if line == "enabled":
local_agent_enabled = "enabled"
local_agent_installed = True
elif line == "disabled":
local_agent_enabled = "disabled"
local_agent_installed = True
elif line in {"active", "inactive", "failed"}:
local_agent_active = line
local_agent_installed = True
blocking_reasons = []
warnings = []
publish_blocking_reasons = []
publish_warnings = []
stack_status = ensure_str(diagnosis.get("stack_status")) or "unknown"
if stack_status == "blocked":
blocking_reasons.append("stack_diagnosis=blocked")
publish_blocking_reasons.append("stack_diagnosis=blocked")
elif stack_status == "attention":
warnings.append("stack_diagnosis=attention")
publish_warnings.append("stack_diagnosis=attention")
elif not truthy_http(stack_status_code):
blocking_reasons.append("stack_diagnosis_unavailable")
publish_blocking_reasons.append("stack_diagnosis_unavailable")
surface_complete = bool(route_surface.get("surface_complete", False))
if not surface_complete:
missing_keys = [ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)]
blocking_reasons.append(
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
)
publish_blocking_reasons.append(
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
)
if not truthy_http(contracts_status_code) or len(contracts) <= 0:
blocking_reasons.append("contracts_unavailable")
publish_blocking_reasons.append("contracts_unavailable")
launchpad_status_code_value = ensure_str(launchpad_status.get("status")) or ensure_str(release_hub.get("launchpad_status")) or ""
if launchpad_status_code_value == "blocked":
blocking_reasons.append("release_launchpad=blocked")
publish_blocking_reasons.append("release_launchpad=blocked")
elif launchpad_status_code_value == "attention":
warnings.append("release_launchpad=attention")
publish_warnings.append("release_launchpad=attention")
elif not truthy_http(launchpad_status_code):
blocking_reasons.append("release_launchpad_unavailable")
publish_blocking_reasons.append("release_launchpad_unavailable")
if launchpad_onboarding_bootstrap_pending_nodes > 0:
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
if launchpad_onboarding_acceptance_ready_nodes > 0:
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
if managed_enabled > 0 and remote_access_ready == 0:
blocking_reasons.append("remote_agent_ready=0")
publish_blocking_reasons.append("remote_agent_ready=0")
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
if queue_dead_letter_nodes > 0:
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
publish_warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
if local_agent_installed and local_agent_active not in {"", "active"}:
warnings.append(f"{local_node_agent_service_name}={local_agent_active}")
log_sync_enabled = bool(log_sync.get("enabled"))
log_sync_state = ensure_str(log_sync.get("state"))
log_sync_mode = ensure_str(log_sync.get("mode"))
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
missing_participating_node_codes = [
ensure_str(item)
for item in ensure_list(log_sync.get("missing_participating_nodes"))
if ensure_str(item)
]
if log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes == 0:
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
elif log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes < participating_nodes_total:
warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
publish_warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
operator_lane = ensure_str(operator_decision.get("lane"))
operator_title = ensure_str(operator_decision.get("title"))
operator_primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
if not operator_lane:
if blocking_reasons:
operator_lane = "recovery"
elif warnings:
operator_lane = "verification"
else:
operator_lane = "go-live"
if not operator_title:
operator_title = (
ensure_str(overview_recommendation.get("title"))
or ensure_str(launchpad_status.get("status_label"))
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
)
next_step_action_code = (
ensure_str(next_step.get("action_code"))
or ensure_str(overview_recommendation.get("primary_action_code"))
or ensure_str(launchpad_status.get("recommended_action_code"))
)
next_step_reason = (
ensure_str(next_step.get("reason"))
or ensure_str(overview_recommendation.get("reason"))
or "来自 launchpad/overview fallback"
)
if not operator_primary_command_key:
operator_primary_command_key = next_step_action_code
go_live_status = "ready"
if blocking_reasons:
go_live_status = "blocked"
elif warnings:
go_live_status = "attention"
publish_status = "ready"
if publish_blocking_reasons:
publish_status = "blocked"
elif publish_warnings:
publish_status = "attention"
publish_ready = publish_status == "ready"
if publish_status == "ready":
publish_status_label = "可发布"
elif publish_status == "attention":
publish_status_label = "可发布但建议先复核"
else:
publish_status_label = "暂不可发布"
action_commands = {
"handover_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} handover_first_gap",
"view_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} view_first_gap",
"bootstrap_run": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {launchpad_recommended_target_node_code} cli"
if launchpad_recommended_target_node_code
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} bootstrap_run"
),
"run_acceptance": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {launchpad_recommended_target_node_code} cli"
if launchpad_recommended_target_node_code
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_acceptance"
),
"fix_managed_nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} fix_managed_nodes",
"open_worker_logs_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} open_worker_logs_participating",
"run_inspection_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_inspection_participating",
}
recommended_commands = {
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary",
"contracts": f"bash domain-api/deploy/multi-region/check_ops_contracts.sh {base_url}",
"ops_plane": f"bash domain-api/deploy/multi-region/check_ops_plane.sh {base_url}",
"release_hub": f"bash domain-api/deploy/multi-region/check_release_hub.sh {base_url}",
"inspection": f"bash domain-api/deploy/multi-region/check_ops_inspection.sh {base_url}",
"node_agent": "bash domain-api/deploy/multi-region/check_node_agent.sh",
"overview": f"bash domain-api/deploy/multi-region/drive_ops_center.sh overview {base_url}",
"doctor_export": (
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url} {overseas_base_url}"
if overseas_base_url
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url}"
),
}
if next_step_action_code and next_step_action_code in action_commands:
recommended_commands["next_step"] = action_commands[next_step_action_code]
if log_sync_enabled:
if "open_worker_logs_participating" in action_commands:
recommended_commands["log_sync_logs"] = action_commands["open_worker_logs_participating"]
if "run_inspection_participating" in action_commands:
recommended_commands["log_sync_inspection"] = action_commands["run_inspection_participating"]
summary = {
"base_url": base_url,
"overseas_base_url": overseas_base_url,
"go_live_status": go_live_status,
"publish_ready": publish_ready,
"publish_status": publish_status,
"publish_status_label": publish_status_label,
"stack_status": stack_status,
"contracts_ready": truthy_http(contracts_status_code) and len(contracts) > 0,
"contracts_total": len(contracts),
"launchpad_status": launchpad_status_code_value,
"launchpad_status_label": ensure_str(launchpad_status.get("status_label")),
"launchpad_recommended_action_code": ensure_str(launchpad_status.get("recommended_action_code")),
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
"route_surface_complete": surface_complete,
"route_surface_missing_keys": [
ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)
],
"managed_enabled": managed_enabled,
"remote_access_ready": remote_access_ready,
"queue_dead_letter_nodes": queue_dead_letter_nodes,
"participating_nodes_total": participating_nodes_total,
"log_sync_enabled": log_sync_enabled,
"log_sync_state": log_sync_state,
"log_sync_mode": log_sync_mode,
"log_sync_covered_nodes": covered_participating_nodes,
"log_sync_missing_node_codes": missing_participating_node_codes,
"next_step_action_code": next_step_action_code,
"next_step_reason": next_step_reason,
"operator_lane": operator_lane,
"operator_title": operator_title,
"operator_primary_command_key": operator_primary_command_key,
"local_node_agent": {
"service_name": local_node_agent_service_name,
"installed": local_agent_installed,
"enabled": local_agent_enabled,
"active": local_agent_active,
},
"blocking_reasons": blocking_reasons,
"warnings": warnings,
"publish_blocking_reasons": publish_blocking_reasons,
"publish_warnings": publish_warnings,
"recommended_commands": recommended_commands,
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,182 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib_ops_center.sh"
ops_center_load_config "${SCRIPT_DIR}"
TARGET_OR_BASE_URL="${1:-}"
SECOND_ARG="${2:-}"
REPORT_DIR=""
MANIFEST_PATH=""
GENERATED_BUNDLE="false"
MAINLAND_BASE_URL=""
OVERSEAS_BASE_URL=""
if [[ -n "${TARGET_OR_BASE_URL}" && ( -f "${TARGET_OR_BASE_URL}" || -d "${TARGET_OR_BASE_URL}" ) ]]; then
if [[ -d "${TARGET_OR_BASE_URL}" ]]; then
REPORT_DIR="${TARGET_OR_BASE_URL%/}"
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
else
MANIFEST_PATH="${TARGET_OR_BASE_URL}"
REPORT_DIR="$(cd "$(dirname "${MANIFEST_PATH}")" && pwd)"
fi
else
MAINLAND_BASE_URL="${TARGET_OR_BASE_URL:-$(ops_center_resolve_mainland_api_base_url)}"
OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${SECOND_ARG:-}" "$(ops_center_resolve_overseas_api_base_url)")"
REPORT_DIR="${OPS_CENTER_REPORT_ROOT}/go-live-bundles/go-live-signoff-$(timestamp_now)"
ops_center_ensure_report_dir "${REPORT_DIR}"
bash "${SCRIPT_DIR}/export_go_live_bundle.sh" "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" >/dev/null
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
GENERATED_BUNDLE="true"
fi
if [[ ! -f "${MANIFEST_PATH}" ]]; then
echo "manifest not found: ${MANIFEST_PATH}" >&2
exit 1
fi
REVIEW_JSON="$(bash "${SCRIPT_DIR}/review_go_live_bundle.sh" "${MANIFEST_PATH}")"
DOCTOR_JSON="$(bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${MANIFEST_PATH}")"
python3 - <<'PY' \
"${REVIEW_JSON}" \
"${DOCTOR_JSON}" \
"${GENERATED_BUNDLE}" \
"${REPORT_DIR}" \
"${MANIFEST_PATH}" \
"${MAINLAND_BASE_URL}" \
"${OVERSEAS_BASE_URL}"
import json
import sys
review = json.loads(sys.argv[1])
doctor = json.loads(sys.argv[2])
generated_bundle = str(sys.argv[3]).strip().lower() == "true"
report_dir = str(sys.argv[4] or "").strip()
manifest_path = str(sys.argv[5] or "").strip()
mainland_base_url = str(sys.argv[6] or "").strip()
overseas_base_url = str(sys.argv[7] or "").strip()
go_live_summary = dict(review.get("go_live_summary") or {})
stack_diagnosis = dict(review.get("stack_diagnosis") or {})
driver_feed = dict(review.get("driver_feed") or {})
codex_brief = dict(review.get("codex_brief") or {})
launchpad_alignment = dict(review.get("launchpad_alignment") or {})
doctor_decision = dict(doctor.get("decision") or {})
doctor_summary = dict(doctor.get("summary") or {})
review_status = str(review.get("status") or "").strip()
doctor_status = str(doctor_decision.get("status") or "").strip()
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
publish_status = str(go_live_summary.get("publish_status") or "").strip()
stack_status = str(stack_diagnosis.get("stack_status") or "").strip()
driver_launch_status = str(driver_feed.get("launch_status") or "").strip()
codex_launch_status = str(codex_brief.get("launch_status") or "").strip()
def dedupe_commands(values):
result = []
seen = set()
for item in values:
text = str(item or "").strip()
if not text or text in seen:
continue
seen.add(text)
result.append(text)
return result
blocked_reasons = dedupe_commands(
[]
+ ([f"review:{review_status}"] if review_status == "blocked" else [])
+ ([f"doctor:{doctor_status}"] if doctor_status == "blocked" else [])
+ ([f"go_live:{go_live_status}"] if go_live_status == "blocked" else [])
+ ([f"publish:{publish_status}"] if publish_status == "blocked" else [])
+ ([f"stack:{stack_status}"] if stack_status == "blocked" else [])
+ ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "blocked" else [])
+ ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "blocked" else [])
)
attention_reasons = dedupe_commands(
[]
+ ([f"review:{review_status}"] if review_status == "attention" else [])
+ ([f"doctor:{doctor_status}"] if doctor_status == "attention" else [])
+ ([f"go_live:{go_live_status}"] if go_live_status == "attention" else [])
+ ([f"publish:{publish_status}"] if publish_status == "attention" else [])
+ ([f"stack:{stack_status}"] if stack_status == "attention" else [])
+ ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "attention" else [])
+ ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "attention" else [])
+ (["launchpad_alignment:inconsistent"] if not bool(launchpad_alignment.get("consistent", False)) else [])
)
if blocked_reasons:
signoff_status = "blocked"
headline = (
str(review.get("headline") or "").strip()
or str(doctor_decision.get("headline") or "").strip()
or "当前发布前验证仍存在阻断项,不应直接签字上线。"
)
elif attention_reasons:
signoff_status = "attention"
headline = (
str(doctor_decision.get("headline") or "").strip()
or str(review.get("headline") or "").strip()
or "当前现场已进入收口区间,但仍建议先完成 attention 项再正式签字。"
)
else:
signoff_status = "ready"
headline = "当前证据包、doctor 结论与发布门禁一致,可进入最终人工签字或正式发布。"
recommended_commands = dedupe_commands(
[]
+ list(doctor_decision.get("recommended_commands") or [])
+ [
f"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review {manifest_path}",
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision {manifest_path}",
]
)
payload = {
"signoff_status": signoff_status,
"headline": headline,
"generated_bundle": generated_bundle,
"report_dir": report_dir or str(doctor.get("report_dir") or review.get("report_dir") or ""),
"manifest_path": manifest_path or str(doctor.get("manifest_path") or ""),
"base_urls": {
"mainland": mainland_base_url,
"overseas": overseas_base_url,
},
"release_gate": {
"go_live_status": go_live_status,
"publish_status": publish_status,
"stack_status": stack_status,
"driver_launch_status": driver_launch_status,
"codex_launch_status": codex_launch_status,
"bundle_review_status": review_status,
"doctor_status": doctor_status,
},
"blocked_reasons": blocked_reasons,
"attention_reasons": attention_reasons,
"decision": {
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
"next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(),
"doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(),
"doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(),
"launchpad_recommended_target_node_code": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"launchpad_recommended_recovery_label": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"launchpad_recommended_recovery_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
},
"launchpad_alignment": launchpad_alignment,
"recommended_commands": recommended_commands,
"bundle_review": review,
"doctor_decision": {
"summary": doctor_summary,
"decision": doctor_decision,
},
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
SERVICE_NAME="${SERVICE_NAME:-domaincheck-node-agent}"
ENV_FILE="${ENV_FILE:-/etc/default/domaincheck-node-agent}"
echo "[1/4] environment file"
if [[ -f "${ENV_FILE}" ]]; then
echo "env file: ${ENV_FILE}"
else
echo "missing env file: ${ENV_FILE}"
fi
echo
echo "[2/4] key env summary"
if [[ -f "${ENV_FILE}" ]]; then
grep -E '^(OPS_CONTROL_PLANE_BASE_URL|NODE_CODE|NODE_REGION|NODE_ROLE|OPS_AGENT_POLL_INTERVAL_SECONDS|WORKER_SERVICE_NAME|API_SERVICE_NAME|SYNC_AGENT_SERVICE_NAME)=' "${ENV_FILE}" || true
else
echo "skip"
fi
echo
echo "[3/4] systemd status"
if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then
systemctl is-enabled "${SERVICE_NAME}" || true
systemctl is-active "${SERVICE_NAME}" || true
else
echo "not-installed"
echo "inactive"
fi
echo
echo "[4/4] service detail"
if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then
systemctl status "${SERVICE_NAME}" --no-pager -l | sed -n '1,25p' || true
else
echo "service not found: ${SERVICE_NAME}"
fi

View File

@@ -0,0 +1,65 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
NODE_CODE="${2:-}"
LIMIT="${3:-80}"
MODE="${4:-key}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-8}"
if [[ -z "${NODE_CODE}" ]]; then
echo "usage: bash domain-api/deploy/multi-region/check_node_scene_log.sh [base_url] <node_code> [limit] [key|full]" >&2
exit 1
fi
RAW_RESPONSE="$(curl -sS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"${BASE_URL}/api/v1/ops/nodes/${NODE_CODE}/scene-log?limit=${LIMIT}&mode=${MODE}")"
echo "[1/2] raw response"
printf '%s\n\n' "${RAW_RESPONSE}"
echo "[2/2] condensed scene-log summary"
python3 - "${RAW_RESPONSE}" <<'PY'
import json
import sys
raw = str(sys.argv[1] or "")
payload = json.loads(raw)
data = dict(payload.get("data") or {})
participation = dict(data.get("participation") or {})
source_summary = dict(data.get("source_summary") or {})
records = list(data.get("records") or [])
latest = dict(data.get("latest_record") or {})
summary = {
"node_code": str(data.get("node_code") or ""),
"status": str(data.get("status") or ""),
"status_label": str(data.get("status_label") or ""),
"mode": str(data.get("mode") or ""),
"summary": str(data.get("summary") or ""),
"log_sync_enabled": bool(data.get("log_sync_enabled", False)),
"records_total": int(data.get("records_total", len(records)) or 0),
"missing_reason_code": str(data.get("missing_reason_code") or ""),
"participation": {
"detect_participating": bool(participation.get("detect_participating", False)),
"participation_state": str(participation.get("participation_state") or ""),
"participation_label": str(participation.get("participation_label") or ""),
"participation_reason": str(participation.get("participation_reason") or ""),
"current_load": int(participation.get("current_load", 0) or 0),
},
"source_summary": {
"line_count": int(source_summary.get("line_count", 0) or 0),
"key_line_count": int(source_summary.get("key_line_count", 0) or 0),
"full_line_count": int(source_summary.get("full_line_count", 0) or 0),
"last_at": str(source_summary.get("last_at") or ""),
"last_line": str(source_summary.get("last_line") or ""),
},
"latest_record": latest,
"sample_messages": [str(item.get("message") or "") for item in records[:8]],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
contracts_json="$(curl -fsS "${BASE_URL}/api/v1/ops/contracts")"
echo "[1/3] ops contracts registry"
printf '%s\n\n' "${contracts_json}"
echo "[2/3] condensed summary"
python3 - <<'PY' "${contracts_json}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
contracts = list(data.get("contracts") or [])
summary = {
"registry_version": str(data.get("registry_version") or ""),
"schema_version": str(data.get("schema_version") or ""),
"contracts_total": len(contracts),
"contract_keys": [str(item.get("key") or "") for item in contracts if str(item.get("key") or "")],
"docs_root": str(data.get("docs_root") or ""),
"discovery_entrypoints": list(data.get("discovery_entrypoints") or []),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo
echo "[3/3] contract rows"
python3 - <<'PY' "${contracts_json}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
contracts = list(data.get("contracts") or [])
rows = [
{
"key": str(item.get("key") or ""),
"version": str(item.get("version") or ""),
"status": str(item.get("status") or ""),
"primary_endpoint": str(item.get("primary_endpoint") or ""),
"schema_doc_path": str(item.get("schema_doc_path") or ""),
"consumers": list(item.get("consumers") or []),
}
for item in contracts
]
print(json.dumps(rows, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,72 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
INSPECTION_JSON="$(curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=120")"
echo "[1/5] inspection overview"
printf '%s\n\n' "${INSPECTION_JSON}"
echo "[2/5] condensed summary"
python3 - <<'PY' "${INSPECTION_JSON}"
import json
import sys
payload = json.loads(sys.argv[1]).get("data") or {}
print(json.dumps({
"rows_total": int(payload.get("rows_total") or 0),
"filtered_rows_total": int(payload.get("filtered_rows_total") or 0),
"visible_nodes_total": int(payload.get("visible_nodes_total") or 0),
"eligible_nodes_total": int(payload.get("eligible_nodes_total") or 0),
"handover_gap_nodes_total": int(payload.get("handover_gap_nodes_total") or 0),
"participating_nodes_total": int(payload.get("participating_nodes_total") or 0),
"summary": dict(payload.get("summary") or {}),
"available_problem_kind_counts": dict(payload.get("available_problem_kind_counts") or {}),
"filters": dict(payload.get("filters") or {}),
}, ensure_ascii=False, indent=2))
PY
echo
echo "[3/5] priority queue"
python3 - <<'PY' "${INSPECTION_JSON}"
import json
import sys
payload = json.loads(sys.argv[1]).get("data") or {}
rows = list(payload.get("priority_queue") or [])
print(json.dumps({
"total": len(rows),
"items": [
{
"node_code": str(item.get("node_code") or ""),
"overall_status": str(item.get("overall_status") or ""),
"problem_kind": str(item.get("problem_kind") or ""),
"problem_label": str(item.get("problem_label") or ""),
"recommended_action_code": str(item.get("recommended_action_code") or ""),
"last_updated_at": str(item.get("last_updated_at") or ""),
"issue_summary": str(item.get("issue_summary") or ""),
}
for item in rows[:12]
],
}, ensure_ascii=False, indent=2))
PY
echo
echo "[4/5] attention only"
curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&status=attention&only_problem=true"
echo
echo
echo "[5/5] participating problems only"
curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&only_problem=true&only_participating=true"
echo

View File

@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
echo "[1/2] ops link snapshot"
SNAPSHOT_JSON="$(curl -fsS "${BASE_URL}/api/v1/ops/link-snapshot")"
echo "${SNAPSHOT_JSON}"
echo
echo
echo "[2/2] condensed summary"
"${PYTHON_BIN}" - <<'PY' "$SNAPSHOT_JSON"
import json
import sys
payload = json.loads(sys.argv[1]).get("data", {})
print(f"status: {payload.get('status', '-')}")
print(f"headline: {payload.get('headline', '-')}")
print()
print("summary:")
for line in payload.get("summary_lines", []) or []:
print(f"- {line}")
actions = payload.get("next_actions", []) or []
if actions:
print()
print("next_actions:")
for item in actions:
label = str(item.get("label", "") or "").strip() or "-"
action_code = str(item.get("action_code", "") or "").strip() or "-"
node_codes = "、".join(str(x).strip() for x in (item.get("node_codes", []) or []) if str(x).strip()) or "-"
print(f"- {label} / {action_code} / nodes: {node_codes}")
PY

View File

@@ -0,0 +1,609 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
curl_post_json() {
local url="$1"
local body="$2"
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
-X POST "${url}" \
-H 'Content-Type: application/json' \
-d "${body}"
}
OVERVIEW_JSON="$(curl_json "${BASE_URL}/api/v1/ops/overview")"
NODES_JSON="$(curl_json "${BASE_URL}/api/v1/ops/nodes")"
JOBS_JSON="$(curl_json "${BASE_URL}/api/v1/ops/jobs")"
ACTIVITY_JSON="$(curl_json "${BASE_URL}/api/v1/ops/activity-stream")"
LAUNCHPAD_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")"
LATEST_RELEASE_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")"
POLICY_PREVIEW_JSON="$(
curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" \
'{"action":"runtime.restart_api","target_type":"node","target_node_code":"mainland-controller-01"}'
)"
echo "[1/12] ops overview"
printf '%s\n\n' "${OVERVIEW_JSON}"
echo "[2/12] top recommendation"
python3 - <<'PY' "${OVERVIEW_JSON}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
recommendation = dict(data.get("recommendation") or {})
print(json.dumps({
"source": str(recommendation.get("source") or ""),
"key": str(recommendation.get("key") or ""),
"priority": str(recommendation.get("priority") or ""),
"summary": str(recommendation.get("summary") or ""),
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
"node_codes": list(recommendation.get("node_codes") or []),
}, ensure_ascii=False, indent=2))
PY
echo
echo "[3/12] driver recommendations (condensed)"
python3 - <<'PY' "${OVERVIEW_JSON}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
recommendations = list(data.get("driver_recommendations") or [])
summary = {
"total": len(recommendations),
"items": [
{
"key": str(item.get("key") or ""),
"title": str(item.get("title") or ""),
"level_label": str(item.get("level_label") or ""),
"primary_action_code": str(item.get("primary_action_code") or ""),
"secondary_action_code": str(item.get("secondary_action_code") or ""),
"meta_text": str(item.get("meta_text") or ""),
}
for item in recommendations[:8]
],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo
echo "[4/12] ops capabilities"
curl_json "${BASE_URL}/api/v1/ops/capabilities"
echo
echo
echo "[5/12] ops blueprint"
curl_json "${BASE_URL}/api/v1/ops/blueprint"
echo
echo
echo "[6/12] ops runbook"
curl_json "${BASE_URL}/api/v1/ops/runbook"
echo
echo
echo "[7/12] ops nodes"
printf '%s\n' "${NODES_JSON}"
echo
echo
echo "[8/12] ops jobs"
printf '%s\n' "${JOBS_JSON}"
echo
echo
echo "[9/12] ops activity stream"
printf '%s\n' "${ACTIVITY_JSON}"
echo
echo
echo "[10/12] release launchpad"
printf '%s\n' "${LAUNCHPAD_JSON}"
echo
echo
echo "[11/12] latest release"
printf '%s\n' "${LATEST_RELEASE_JSON}"
echo
echo
echo "[12/12] policy preview sample"
printf '%s\n' "${POLICY_PREVIEW_JSON}"
echo
echo
echo "[13/13] condensed summary"
python3 - <<'PY' \
"${BASE_URL}" \
"${OVERVIEW_JSON}" \
"${NODES_JSON}" \
"${JOBS_JSON}" \
"${ACTIVITY_JSON}" \
"${LAUNCHPAD_JSON}" \
"${LATEST_RELEASE_JSON}" \
"${POLICY_PREVIEW_JSON}"
import json
import sys
def as_dict(payload):
try:
raw = json.loads(payload)
except Exception:
return {}
return raw.get("data") or {}
def ensure_dict(value):
return value if isinstance(value, dict) else {}
def ensure_list(value):
return value if isinstance(value, list) else []
def ensure_str(value):
return value if isinstance(value, str) else ""
def command(name, *parts):
tokens = [
"bash",
"domain-api/deploy/multi-region/drive_ops_center.sh",
name,
base_url,
]
tokens.extend([str(part) for part in parts if str(part or "").strip()])
return " ".join(tokens)
def release_command(name, *parts):
tokens = [
"bash",
"domain-api/deploy/multi-region/drive_release_hub.sh",
base_url,
name,
]
tokens.extend([str(part) for part in parts if str(part or "").strip()])
return " ".join(tokens)
def build_operator_decision():
remote_access_state_counts = ensure_dict(node_summary.get("remote_access_state_counts"))
pending_bootstrap = int(node_summary.get("pending_bootstrap", 0) or 0)
runtime_only = int(node_summary.get("runtime_only", 0) or 0)
online_total = int(node_summary.get("online", 0) or 0)
participating_total = int(scene_log.get("participating_node_count", 0) or 0)
launchpad_blocked = bool(launchpad_workers.get("blocked")) or bool(launchpad_controls.get("blocked"))
running_total = int(jobs_summary.get("running_count", 0) or 0)
queued_total = int(jobs_summary.get("queued_count", 0) or 0)
if scene_log.get("status") == "disabled" and participating_total > 0:
return {
"lane": "observability",
"priority": "high",
"reason_code": "scene_log_disabled",
"title": "先恢复远端现场日志可见性",
"summary": ensure_str(scene_log.get("summary")),
"next_focus": ensure_dict(scene_log.get("focus_ref")),
"primary_command_key": "scene_log",
"secondary_command_key": "driver_focus_preview",
}
if pending_bootstrap > 0 or remote_access_state_counts:
return {
"lane": "node_handover",
"priority": "high",
"reason_code": "managed_nodes_not_ready",
"title": "先补齐节点接管与远程可达性",
"summary": (
f"当前 managed node 共 {int(node_summary.get('managed_total', 0) or 0)} 台,"
f"pending_bootstrap={pending_bootstrap}runtime_only={runtime_only}。"
),
"next_focus": {
"kind": "managed_node_handover",
"node_code": first_attention_node_code,
} if first_attention_node_code else {},
"primary_command_key": "node_handover",
"secondary_command_key": "node_bootstrap_plan",
}
if launchpad_blocked or str(launchpad_status.get("status") or "") == "attention":
return {
"lane": "release",
"priority": "medium",
"reason_code": "release_launchpad_attention",
"title": "检查发布驾驶舱与 rollout 阻断",
"summary": ensure_str(launchpad_status.get("summary")),
"next_focus": release_focus_ref,
"primary_command_key": "release_launchpad",
"secondary_command_key": "release_preview_latest",
}
if running_total > 0 or queued_total > 0:
return {
"lane": "ops_jobs",
"priority": "medium",
"reason_code": "jobs_active",
"title": "继续盯运行中任务与活动流",
"summary": (
f"当前 running={running_total}queued={queued_total}"
f"activity_status={ensure_str(activity_summary.get('status')) or 'unknown'}。"
),
"next_focus": {
"kind": "ops_jobs",
"section": "activity_stream",
},
"primary_command_key": "jobs",
"secondary_command_key": "activity_preview",
}
return {
"lane": "steady",
"priority": "low",
"reason_code": "steady_state",
"title": "当前进入稳定观察态",
"summary": (
f"online={online_total}attention_nodes={len(attention_nodes)}"
f"scene_log_status={ensure_str(scene_log.get('status')) or 'standby'}。"
),
"next_focus": recommendation_focus_ref or ensure_dict(scene_log.get("focus_ref")),
"primary_command_key": "driver_feed",
"secondary_command_key": "activity_preview",
}
def normalize_scene_log_observation(overview_data):
driver_feed = ensure_dict(overview_data.get("driver_feed"))
scene_log_data = ensure_dict(driver_feed.get("scene_log_observation"))
if scene_log_data:
normalized = dict(scene_log_data)
normalized["status"] = ensure_str(normalized.get("status")) or "standby"
normalized["status_label"] = ensure_str(normalized.get("status_label")) or "待命"
normalized["summary"] = ensure_str(normalized.get("summary")) or "当前现场日志观察暂无额外结论。"
normalized["focus_ref"] = ensure_dict(normalized.get("focus_ref"))
normalized["target_node_code"] = ensure_str(normalized.get("target_node_code"))
normalized["participating_node_count"] = int(normalized.get("participating_node_count", 0) or 0)
normalized["covered_participating_node_count"] = int(normalized.get("covered_participating_node_count", 0) or 0)
normalized["line_count"] = int(normalized.get("line_count", 0) or 0)
return normalized
execution_scene = ensure_dict(overview_data.get("execution_scene"))
log_sync = ensure_dict(execution_scene.get("log_sync"))
participation = ensure_dict(execution_scene.get("participation_summary"))
missing_nodes = ensure_list(log_sync.get("missing_participating_nodes"))
participating_nodes = ensure_list(execution_scene.get("participating_nodes"))
target_node_code = ""
if missing_nodes:
target_node_code = ensure_str(missing_nodes[0])
elif participating_nodes:
target_node_code = ensure_str(ensure_dict(participating_nodes[0]).get("node_code"))
participating_total = int(
log_sync.get("participating_node_count", participation.get("participating_nodes", len(participating_nodes)))
or participation.get("participating_nodes", len(participating_nodes))
or len(participating_nodes)
or 0
)
covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0)
line_count = int(log_sync.get("line_count", 0) or 0)
enabled = bool(log_sync.get("enabled", False))
mode = ensure_str(log_sync.get("mode")) or "key"
status = "standby"
status_label = "待命"
summary = "当前没有参与检测节点,现场日志观察处于待命状态。"
if participating_total > 0 and not enabled:
status = "disabled"
status_label = "未开启"
summary = (
f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭,"
"海外控制面还看不到节点级现场日志。"
)
elif participating_total > 0 and (
line_count <= 0 or covered_total <= 0 or int(log_sync.get("missing_participating_node_count", len(missing_nodes)) or len(missing_nodes)) > 0
):
status = "waiting_sample"
status_label = "等待样本"
summary = (
f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total}"
f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。"
)
elif participating_total > 0:
status = "ready"
status_label = "可下钻"
summary = (
f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点,"
"可以直接下钻节点现场日志。"
)
focus_ref = {}
if target_node_code:
focus_ref = {
"kind": "node_scene_log",
"node_code": target_node_code,
"mode": mode,
"limit": 120 if status == "waiting_sample" else 80,
}
return {
"status": status,
"status_label": status_label,
"summary": summary,
"focus_ref": focus_ref,
"target_node_code": target_node_code,
"participating_node_count": participating_total,
"covered_participating_node_count": covered_total,
"line_count": line_count,
}
base_url = ensure_str(sys.argv[1]) or "http://127.0.0.1:8100"
overview = as_dict(sys.argv[2])
nodes = as_dict(sys.argv[3])
jobs = as_dict(sys.argv[4])
activity = as_dict(sys.argv[5])
launchpad = as_dict(sys.argv[6])
latest_release = as_dict(sys.argv[7])
policy_preview = as_dict(sys.argv[8])
recommendation = ensure_dict(overview.get("recommendation"))
scene_log = normalize_scene_log_observation(overview)
managed_nodes = ensure_list(nodes.get("managed_nodes"))
node_summary = ensure_dict(nodes.get("summary"))
jobs_summary = ensure_dict(jobs.get("summary"))
activity_summary = ensure_dict(activity.get("summary"))
launchpad_release = ensure_dict(launchpad.get("release"))
launchpad_rec = ensure_dict(launchpad.get("top_recommendation"))
launchpad_workers = ensure_dict(launchpad.get("worker_rollout_preview"))
launchpad_controls = ensure_dict(launchpad.get("control_rollout_preview"))
launchpad_status = ensure_dict(launchpad.get("launchpad_status"))
latest_release_entry = latest_release if isinstance(latest_release, dict) else {}
latest_package_preview = ensure_dict(launchpad.get("latest_package_preview"))
online_nodes = [
node for node in managed_nodes
if str(node.get("status") or "").lower() in {"online", "busy"}
]
attention_nodes = [
{
"node_code": str(node.get("node_code") or ""),
"status": str(node.get("status") or ""),
"role": str(node.get("role") or ""),
"stack_status_label": str(node.get("stack_status_label") or ""),
"recommended_action_code": str(node.get("recommended_action_code") or ""),
"recommended_action_label": str(node.get("recommended_action_label") or ""),
"primary_issue": str(node.get("primary_issue") or ""),
}
for node in managed_nodes
if str(node.get("status") or "").lower() not in {"online", "busy"}
or str(node.get("stack_status") or "").lower() not in {"healthy", "ready", ""}
]
activity_items = ensure_list(activity.get("items"))
recent_activity = [
{
"kind": str(item.get("kind") or ""),
"title": str(item.get("title") or ""),
"level": str(item.get("level") or ""),
"created_at": str(item.get("created_at") or ""),
}
for item in activity_items[:5]
]
job_items = ensure_list(jobs.get("items"))
running_jobs = [
{
"job_key": str(item.get("job_key") or ""),
"status": str(item.get("status") or ""),
"title": str(item.get("title") or ""),
"progress_text": str(item.get("progress_text") or ""),
}
for item in job_items
if str(item.get("status") or "").lower() in {"running", "queued", "waiting"}
][:5]
recommendation_focus_ref = ensure_dict(recommendation.get("focus_ref"))
first_attention = attention_nodes[0] if attention_nodes else {}
first_attention_node_code = ensure_str(first_attention.get("node_code"))
scene_log_target_node_code = ensure_str(scene_log.get("target_node_code"))
latest_release_version = str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or "")
release_focus_ref = ensure_dict(launchpad_rec.get("focus_ref"))
release_id = str(release_focus_ref.get("release_id") or "")
operator_decision = build_operator_decision()
primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
secondary_command_key = ensure_str(operator_decision.get("secondary_command_key"))
result = {
"overview": {
"top_recommendation": {
"source": str(recommendation.get("source") or ""),
"key": str(recommendation.get("key") or ""),
"priority": str(recommendation.get("priority") or ""),
"summary": str(recommendation.get("summary") or ""),
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
"focus_ref": recommendation.get("focus_ref") or {},
"node_codes": list(recommendation.get("node_codes") or []),
},
"scene_log_observation": {
"status": str(scene_log.get("status") or ""),
"status_label": str(scene_log.get("status_label") or ""),
"summary": str(scene_log.get("summary") or ""),
"target_node_code": str(scene_log.get("target_node_code") or ""),
"participating_node_count": int(scene_log.get("participating_node_count") or 0),
"covered_participating_node_count": int(scene_log.get("covered_participating_node_count") or 0),
"line_count": int(scene_log.get("line_count") or 0),
"focus_ref": scene_log.get("focus_ref") or {},
},
},
"nodes": {
"managed_total": int(node_summary.get("managed_total") or len(managed_nodes)),
"online_or_busy_total": len(online_nodes),
"offline_total": int(node_summary.get("offline_count") or 0),
"attention_total": len(attention_nodes),
"attention_nodes": attention_nodes[:8],
},
"jobs": {
"running_total": int(jobs_summary.get("running_count") or 0),
"queued_total": int(jobs_summary.get("queued_count") or 0),
"failed_total": int(jobs_summary.get("failed_count") or 0),
"recent_running_jobs": running_jobs,
},
"activity_stream": {
"total": int(activity_summary.get("total") or len(activity_items)),
"error_count": int(activity_summary.get("error_count") or 0),
"warning_count": int(activity_summary.get("warning_count") or 0),
"running_job_count": int(activity_summary.get("running_job_count") or 0),
"recent_items": recent_activity,
},
"operator_decision": {
"lane": ensure_str(operator_decision.get("lane")),
"priority": ensure_str(operator_decision.get("priority")),
"reason_code": ensure_str(operator_decision.get("reason_code")),
"title": ensure_str(operator_decision.get("title")),
"summary": ensure_str(operator_decision.get("summary")),
"next_focus": ensure_dict(operator_decision.get("next_focus")),
"primary_command_key": primary_command_key,
"secondary_command_key": secondary_command_key,
},
"recommended_commands": {
"stack_diagnosis": "bash domain-api/deploy/multi-region/check_ops_center_stack.sh",
"driver_feed": command("driver-feed"),
"activity_preview": command("activity-preview"),
"driver_focus_preview": command("driver-focus-preview"),
"driver_focus_run": command("driver-focus-run"),
"top_recommendation_preview": (
command("driver-preview", str(recommendation.get("primary_action_code") or ""))
if str(recommendation.get("primary_action_code") or "").strip()
else ""
),
"top_recommendation_focus_preview": (
command("driver-focus-preview", f"entry:{str(recommendation.get('key') or '')}")
if str(recommendation.get("key") or "").strip()
else ""
),
"scene_log": (
command("scene-node-log", scene_log_target_node_code, "120", "key")
if scene_log_target_node_code
else ""
),
"scene_log_direct": (
f"bash domain-api/deploy/multi-region/check_node_scene_log.sh {base_url} {scene_log_target_node_code} 120 key"
if scene_log_target_node_code
else ""
),
"node_handover": (
command("node-handover", first_attention_node_code)
if first_attention_node_code
else command("stack-first-gap-handover")
),
"node_bootstrap_plan": (
command("node-bootstrap-plan", first_attention_node_code)
if first_attention_node_code
else ""
),
"delivery_queue": (
command("delivery-queue", first_attention_node_code)
if first_attention_node_code
else ""
),
"release_launchpad": release_command("launchpad"),
"release_preview_latest": release_command("preview-latest"),
"release_preview_latest_worker": release_command("preview-latest-smart", "worker"),
"release_preview_latest_control": release_command("preview-latest-smart", "control"),
"release_focus": (
command("driver-preview", "focus_release_hub")
if str(launchpad_rec.get("action_code") or "").strip()
else ""
),
"release_preview_selected": (
release_command("preview-release", release_id, "worker")
if release_id
else ""
),
"jobs": command("activity-stream", "kind=ops_job"),
},
"next_actions": {
"primary_command": "",
"secondary_command": "",
},
"release_launchpad": {
"status": str(launchpad_status.get("status") or ""),
"status_label": str(launchpad_status.get("status_label") or ""),
"summary": str(launchpad_status.get("summary") or ""),
"recommended_action_code": str(launchpad_rec.get("action_code") or ""),
"recommended_action_label": str(launchpad_rec.get("action_label") or ""),
"recommended_execution_mode": str(launchpad_rec.get("execution_mode") or ""),
"recommended_execution_mode_label": str(launchpad_rec.get("execution_mode_label") or ""),
"focus_ref": launchpad_rec.get("focus_ref") or {},
"release_version": str(launchpad_release.get("version") or launchpad_release.get("release_version") or ""),
"worker_rollout_preview": {
"available": bool(launchpad_workers.get("available")),
"blocked": bool(launchpad_workers.get("blocked")),
"execution_mode": str(launchpad_workers.get("execution_mode") or ""),
"execution_mode_label": str(launchpad_workers.get("execution_mode_label") or ""),
"target_node_codes": list(launchpad_workers.get("target_node_codes") or []),
},
"control_rollout_preview": {
"available": bool(launchpad_controls.get("available")),
"blocked": bool(launchpad_controls.get("blocked")),
"execution_mode": str(launchpad_controls.get("execution_mode") or ""),
"execution_mode_label": str(launchpad_controls.get("execution_mode_label") or ""),
"target_node_codes": list(launchpad_controls.get("target_node_codes") or []),
},
},
"latest_release": {
"version": str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or ""),
"status": ensure_str(latest_release_entry.get("status")),
"status_label": ensure_str(latest_release_entry.get("status_label")),
"summary": ensure_str(latest_release_entry.get("summary") or latest_release_entry.get("summary_text")),
},
"policy_preview_sample": {
"execution_mode": str(policy_preview.get("execution_mode") or ""),
"risk_level": str(policy_preview.get("risk_level") or ""),
"approval_required": bool(policy_preview.get("approval_required")),
"blocked": bool(policy_preview.get("blocked")),
"blocking_reasons": ensure_list(policy_preview.get("blocking_reasons"))[:5],
"warnings": ensure_list(policy_preview.get("warnings"))[:5],
},
"focus_refs": {
"top_recommendation": recommendation_focus_ref,
"scene_log": ensure_dict(scene_log.get("focus_ref")),
"release_hub": release_focus_ref,
},
"rollout_targets": {
"worker": list(launchpad_workers.get("target_node_codes") or []),
"control": list(launchpad_controls.get("target_node_codes") or []),
},
"package_preview": {
"available": bool(latest_package_preview.get("release")),
"preview_source": ensure_str(latest_package_preview.get("preview_source")),
"message": ensure_str(latest_package_preview.get("message")),
},
}
result["next_actions"]["primary_command"] = ensure_str(
ensure_dict(result.get("recommended_commands")).get(primary_command_key)
)
result["next_actions"]["secondary_command"] = ensure_str(
ensure_dict(result.get("recommended_commands")).get(secondary_command_key)
)
print(json.dumps(result, ensure_ascii=False, indent=2))
PY
echo

View File

@@ -0,0 +1,373 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
INPUT_RELEASE_ID="${2:-}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
curl_post_json() {
local url="$1"
local body="$2"
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
-X POST "${url}" \
-H 'Content-Type: application/json' \
-d "${body}"
}
launchpad="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")"
latest_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")"
action_templates="$(curl_json "${BASE_URL}/api/v1/ops/action-templates")"
cluster_json="$(curl_json "${BASE_URL}/api/v1/runtime/cluster")"
release_id="${INPUT_RELEASE_ID}"
if [[ -z "${release_id}" ]]; then
release_id="$(python3 -c '
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
value = data.get("id") or 0
print(value)
' "${latest_release}")"
fi
selected_release="${latest_release}"
if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then
selected_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}")"
fi
build_release_preview_payload() {
local preview_mode="$1"
python3 - <<'PY' "${selected_release}" "${cluster_json}" "${preview_mode}"
import json
import sys
release_payload = json.loads(sys.argv[1]).get("data") or {}
cluster_payload = json.loads(sys.argv[2]).get("data") or {}
preview_mode = str(sys.argv[3] or "").strip().lower()
nodes = list(cluster_payload.get("nodes") or [])
def is_online(node):
return str(node.get("status") or "").strip() in {"online", "busy"}
control_nodes = [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node) and str(node.get("role") or "").strip() == "control"
]
worker_nodes = [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node)
and str(node.get("role") or "").strip() == "worker"
and bool(node.get("is_effective_worker", False))
]
online_nodes = [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node)
]
payload = {
"release_version": str(release_payload.get("release_version") or "").strip(),
"artifact_url": str(release_payload.get("artifact_url") or "").strip(),
"checksum": str(release_payload.get("checksum") or "").strip(),
"install_root": "/opt/domaincheck",
"health_check_timeout_seconds": 10,
"health_check_retries": 2,
"health_check_interval_seconds": 2,
"rollback_on_failure": True,
"switch_current": True,
}
if preview_mode == "control":
template_key = "deploy.release.control"
target_node_codes = control_nodes
payload.update(
{
"restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
"health_check_urls": ["http://127.0.0.1:8100/health"],
"health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
}
)
elif preview_mode == "worker":
template_key = "deploy.release.worker"
target_node_codes = worker_nodes
payload.update(
{
"restart_services": ["domaincheck-worker"],
"health_check_urls": [],
"health_check_services": ["domaincheck-worker"],
}
)
else:
template_key = "deploy.release.custom"
target_node_codes = (control_nodes[:1] + worker_nodes[:2]) or online_nodes[:3]
payload.update(
{
"restart_services": [],
"health_check_urls": [],
"health_check_services": [],
}
)
request_payload = {
"template_key": template_key,
"action": "deploy.release",
"target_type": "batch",
"execution_mode": "remote-agent",
"target_node_codes": [item for item in target_node_codes if item],
"payload": payload,
}
print(json.dumps(request_payload, ensure_ascii=False))
PY
}
print_preview_request_summary() {
local preview_payload="$1"
python3 - <<'PY' "${preview_payload}"
import json
import sys
payload = json.loads(sys.argv[1])
body = payload.get("payload") or {}
print(
json.dumps(
{
"template_key": payload.get("template_key") or "",
"action": payload.get("action") or "",
"execution_mode": payload.get("execution_mode") or "",
"target_node_codes": payload.get("target_node_codes") or [],
"release_version": body.get("release_version") or "",
"artifact_url_present": bool(str(body.get("artifact_url") or "").strip()),
"checksum_present": bool(str(body.get("checksum") or "").strip()),
"restart_services": body.get("restart_services") or [],
"health_check_urls": body.get("health_check_urls") or [],
"health_check_services": body.get("health_check_services") or [],
},
ensure_ascii=False,
indent=2,
)
)
PY
}
run_release_preview() {
local step_no="$1"
local preview_mode="$2"
local section_title="$3"
local preview_payload
preview_payload="$(build_release_preview_payload "${preview_mode}")"
echo "[${step_no}/10] ${section_title}"
print_preview_request_summary "${preview_payload}"
echo
curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" "${preview_payload}"
echo
echo
}
echo "[1/11] release launchpad"
printf '%s\n\n' "${launchpad}"
echo "[2/11] latest release"
printf '%s\n\n' "${selected_release}"
echo "[3/11] recent releases"
curl_json "${BASE_URL}/api/v1/ops/releases?limit=8"
echo
echo
echo "[4/11] release deploy templates"
python3 - <<'PY' "${action_templates}"
import json
import sys
payload = json.loads(sys.argv[1])
groups = (payload.get("data") or {}).get("groups") or []
release_group = next((item for item in groups if str(item.get("group_key") or "") == "release"), {})
items = list(release_group.get("items") or [])
summary = {
"group_key": release_group.get("group_key") or "",
"group_title": release_group.get("group_title") or "",
"template_keys": [str(item.get("key") or "") for item in items],
"field_types": sorted(
{
str(field.get("type") or "")
for item in items
for field in list(item.get("fields") or [])
if str(field.get("type") or "")
}
),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo
echo "[5/11] online release target summary"
python3 - <<'PY' "${cluster_json}"
import json
import sys
payload = json.loads(sys.argv[1]).get("data") or {}
nodes = list(payload.get("nodes") or [])
def is_online(node):
return str(node.get("status") or "").strip() in {"online", "busy"}
summary = {
"online_control_nodes": [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node) and str(node.get("role") or "").strip() == "control"
],
"online_worker_nodes": [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False))
],
"all_online_nodes": [
{
"node_code": str(node.get("node_code") or "").strip(),
"role": str(node.get("role") or "").strip(),
"status": str(node.get("status") or "").strip(),
"is_effective_worker": bool(node.get("is_effective_worker", False)),
}
for node in nodes
if is_online(node)
],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo
echo
run_release_preview "6" "control" "deploy.release.control preview"
run_release_preview "7" "worker" "deploy.release.worker preview"
run_release_preview "8" "custom" "deploy.release.custom preview"
rollout_list='{"code":0,"message":"no release selected","data":{"rollouts":[]}}'
rollout_id="0"
if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then
rollout_list="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}/rollouts?limit=10")"
rollout_id="$(python3 -c '
import json
import sys
payload = json.loads(sys.argv[1])
rollouts = (payload.get("data") or {}).get("rollouts") or []
print((rollouts[0] or {}).get("id") or 0 if rollouts else 0)
' "${rollout_list}")"
fi
echo "[9/11] rollout list for release ${release_id:-0}"
printf '%s\n\n' "${rollout_list}"
echo "[10/11] primary rollout detail"
if [[ "${rollout_id}" != "0" ]]; then
curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}"
else
echo '{"code":0,"message":"no rollout","data":{}}'
fi
echo
echo
echo "[11/12] primary rollout jobs"
if [[ "${rollout_id}" != "0" ]]; then
curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}/jobs?limit=50"
else
echo '{"code":0,"message":"no rollout","data":{"jobs":[]}}'
fi
echo
echo
echo "[12/12] condensed summary"
python3 - <<'PY' "${launchpad}" "${selected_release}" "${cluster_json}" "${rollout_list}" "${rollout_id}"
import json
import sys
launchpad_payload = json.loads(sys.argv[1] or "{}").get("data") or {}
selected_release = json.loads(sys.argv[2] or "{}").get("data") or {}
cluster_payload = json.loads(sys.argv[3] or "{}").get("data") or {}
rollout_list_payload = json.loads(sys.argv[4] or "{}").get("data") or {}
rollout_id = int(sys.argv[5] or 0)
launchpad_status = dict(launchpad_payload.get("launchpad_status") or {})
worker_preview = dict(launchpad_payload.get("worker_rollout_preview") or {})
worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {})
control_preview = dict(launchpad_payload.get("control_rollout_preview") or {})
control_smart_rollout = dict(control_preview.get("smart_rollout") or {})
nodes = list(cluster_payload.get("nodes") or [])
rollouts = list(rollout_list_payload.get("rollouts") or [])
def is_online(node):
return str(node.get("status") or "").strip() in {"online", "busy"}
summary = {
"release": {
"id": int(selected_release.get("id") or 0),
"release_version": str(selected_release.get("release_version") or ""),
"channel": str(selected_release.get("channel") or ""),
"status": str(selected_release.get("status") or ""),
},
"launchpad": {
"status": str(launchpad_status.get("status") or ""),
"status_label": str(launchpad_status.get("status_label") or ""),
"summary": str(launchpad_status.get("summary") or ""),
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
"recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""),
"recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""),
"recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""),
"onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
"onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
"recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""),
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
},
"worker_rollout_preview": {
"available": bool(worker_smart_rollout.get("available", False)),
"blocked": bool(worker_preview.get("blocked", False)),
"requires_confirmation": bool(worker_preview.get("requires_confirmation", False)),
"execution_mode": str(worker_smart_rollout.get("execution_mode") or ""),
"target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []),
},
"control_rollout_preview": {
"available": bool(control_smart_rollout.get("available", False)),
"blocked": bool(control_preview.get("blocked", False)),
"requires_confirmation": bool(control_preview.get("requires_confirmation", False)),
"execution_mode": str(control_smart_rollout.get("execution_mode") or ""),
"target_node_codes": list(control_smart_rollout.get("target_node_codes") or []),
},
"cluster": {
"online_control_nodes": [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node) and str(node.get("role") or "").strip() == "control"
],
"online_worker_nodes": [
str(node.get("node_code") or "").strip()
for node in nodes
if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False))
],
},
"rollouts": {
"count": len(rollouts),
"selected_rollout_id": rollout_id,
},
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo

View File

@@ -4,15 +4,24 @@ set -euo pipefail
MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
PYTHON_BIN="${PYTHON_BIN:-python3}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
echo "[1/5] mainland runtime cluster"
MAINLAND_CLUSTER="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
MAINLAND_CLUSTER="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
echo "${MAINLAND_CLUSTER}"
echo
echo
echo "[2/5] mainland runtime readiness"
MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
MAINLAND_READINESS="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
echo "${MAINLAND_READINESS}"
echo
echo
@@ -79,7 +88,7 @@ echo
echo
echo "[4/5] overseas runtime cluster"
OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
OVERSEAS_CLUSTER="$(curl_json "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
echo "${OVERSEAS_CLUSTER}"
echo
echo

View File

@@ -4,23 +4,44 @@ set -euo pipefail
DB_NAME="${DB_NAME:-domain}"
DB_USER="${DB_USER:-postgres}"
PSQL_BIN="${PSQL_BIN:-/www/server/pgsql/bin/psql}"
API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:8100}}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
echo "[1/3] cluster nodes"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
RUNTIME_STATUS_JSON="$(curl_json "${API_BASE_URL}/api/v1/runtime/status")"
echo "[1/4] cluster nodes"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
select
node_code || '|' ||
role || '|' ||
status || '|' ||
coalesce(current_load, 0) || '|' ||
coalesce(metadata_json->>'job_items_claimed', '0') || '|' ||
coalesce(metadata_json->>'job_items_running', '0') || '|' ||
coalesce(metadata_json->>'job_items_completed', '0') || '|' ||
to_char(last_heartbeat_at, 'YYYY-MM-DD HH24:MI:SS') || '|' ||
coalesce(metadata_json->>'worker_online', '') || '|' ||
coalesce(metadata_json->>'detect_participating', '')
(
case
when coalesce(current_load, 0) > 0 then 'true'
when coalesce(metadata_json->>'detect_participating', '') in ('true', 'false') then metadata_json->>'detect_participating'
else ''
end
)
from detect_worker_nodes
order by node_code;
"
echo
echo "[2/3] active job distribution"
echo "[2/4] active job distribution"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
select
coalesce(claimed_by, '') || '|' ||
@@ -39,59 +60,43 @@ order by claimed_by, status;
"
echo
echo "[3/3] condensed summary"
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -At <<'SQL'
with latest_job as (
select id, job_code, status
from detect_jobs
where status in ('pending','running')
order by id desc
limit 1
),
node_stats as (
select
node_code,
role,
status,
coalesce(metadata_json->>'worker_online', '') as worker_online,
coalesce(metadata_json->>'detect_participating', '') as detect_participating
from detect_worker_nodes
),
item_stats as (
select
coalesce(claimed_by, '') as claimed_by,
status,
count(*) as cnt
from detect_job_items
where job_id = (select id from latest_job)
group by claimed_by, status
)
select json_build_object(
'job', (
select json_build_object(
'job_code', coalesce(job_code, ''),
'status', coalesce(status, '')
)
from latest_job
),
'online_nodes', (
select coalesce(json_agg(json_build_object(
'node_code', node_code,
'role', role,
'status', status,
'worker_online', worker_online,
'detect_participating', detect_participating
) order by node_code), '[]'::json)
from node_stats
where status in ('online', 'busy')
),
'claimed_distribution', (
select coalesce(json_agg(json_build_object(
'claimed_by', claimed_by,
'status', status,
'count', cnt
) order by claimed_by, status), '[]'::json)
from item_stats
)
);
SQL
echo "[3/4] runtime participation snapshot"
printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c '
import json, sys
payload = json.load(sys.stdin).get("data", {})
detect = payload.get("detect", {})
print(json.dumps({
"node": payload.get("node", {}),
"participation_summary": detect.get("participation_summary", {}),
"participating_nodes": detect.get("participating_nodes", []),
"non_participating_nodes": detect.get("non_participating_nodes", detect.get("standby_nodes", [])),
"log_sync": detect.get("log_sync", {}),
}, ensure_ascii=False, indent=2))
'
echo
echo "[4/4] condensed summary"
printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c '
import json, sys
payload = json.load(sys.stdin).get("data", {})
detect = payload.get("detect", {})
summary = detect.get("participation_summary", {}) or {}
print(json.dumps({
"node": payload.get("node", {}),
"active_job": {
"job_code": (detect.get("active_job") or {}).get("job_code", ""),
"status": (detect.get("active_job") or {}).get("status", ""),
},
"participation_summary": summary,
"dispatch_active_nodes": summary.get("dispatch_active_node_codes", []),
"recent_only_nodes": summary.get("recent_only_node_codes", []),
"non_participating_nodes": summary.get("non_participating_node_codes", []),
"log_sync": {
"enabled": bool((detect.get("log_sync") or {}).get("enabled", False)),
"mode": (detect.get("log_sync") or {}).get("mode", ""),
"line_count": (detect.get("log_sync") or {}).get("line_count", 0),
"source_nodes": (detect.get("log_sync") or {}).get("source_nodes", []),
"last_at": (detect.get("log_sync") or {}).get("last_at", ""),
},
}, ensure_ascii=False, indent=2))
'

View File

@@ -0,0 +1,447 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib_ops_center.sh"
ops_center_load_config "${SCRIPT_DIR}"
usage() {
cat <<'EOF'
usage:
bash domain-api/deploy/multi-region/drive_ops_action.sh <base_url> <preview|execute> <action_code> [action_payload_json|@payload.json] [requested_by]
bash domain-api/deploy/multi-region/drive_ops_action.sh <base_url> <action_code> [action_payload_json|@payload.json] [requested_by]
examples:
bash domain-api/deploy/multi-region/drive_ops_action.sh \
http://127.0.0.1:8100 \
preview \
create_release_rollout_worker \
'{"release_id":12}'
bash domain-api/deploy/multi-region/drive_ops_action.sh \
http://127.0.0.1:8100 \
create_release_rollout_worker \
'{"release_id":12}'
NODE_CODES="mainland-worker-01,mainland-worker-02" \
bash domain-api/deploy/multi-region/drive_ops_action.sh \
http://127.0.0.1:8100 \
run_standard_inspection \
'{}'
notes:
- preview/execute 双模式共用同一份 action payload 和 node_codes
- 旧用法仍默认按 execute 处理
- NODE_CODES can be passed through env as comma/newline separated node_code list
- action_payload_json can also use @/path/to/file.json
- preview metadata can be passed through env:
SOURCE_LABEL / ACTION_TITLE / ACTION_SUMMARY / ACTION_REASON / ACTION_META_TEXT / PRIMARY_LABEL
EOF
}
BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}"
SECOND_ARG="${2:-}"
THIRD_ARG="${3:-}"
FOURTH_ARG="${4:-}"
FIFTH_ARG="${5:-}"
if [[ "${SECOND_ARG}" == "preview" || "${SECOND_ARG}" == "execute" ]]; then
OPERATION="${SECOND_ARG}"
ACTION_CODE="${THIRD_ARG:-}"
RAW_ACTION_PAYLOAD="${FOURTH_ARG:-}"
REQUESTED_BY="${FIFTH_ARG:-$(ops_center_resolve_requested_by)}"
else
OPERATION="execute"
ACTION_CODE="${SECOND_ARG:-}"
RAW_ACTION_PAYLOAD="${THIRD_ARG:-}"
REQUESTED_BY="${FOURTH_ARG:-$(ops_center_resolve_requested_by)}"
fi
if [[ -z "${RAW_ACTION_PAYLOAD}" ]]; then
RAW_ACTION_PAYLOAD='{}'
fi
NODE_CODES_INPUT="${NODE_CODES:-}"
SOURCE_LABEL="${SOURCE_LABEL:-cli/drive_ops_action}"
ACTION_TITLE="${ACTION_TITLE:-}"
ACTION_SUMMARY="${ACTION_SUMMARY:-}"
ACTION_REASON="${ACTION_REASON:-}"
ACTION_META_TEXT="${ACTION_META_TEXT:-}"
PRIMARY_LABEL="${PRIMARY_LABEL:-}"
print_json_or_raw() {
local raw_text="${1:-}"
python3 - "${raw_text}" <<'PY'
import json
import sys
raw_text = str(sys.argv[1] or "")
try:
payload = json.loads(raw_text)
except Exception:
print(raw_text)
else:
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY
}
request_with_status() {
local method="$1"
local url="$2"
local body="${3:-}"
if [[ "${method}" == "POST" ]]; then
curl -sS -X POST "${url}" \
-H 'Content-Type: application/json' \
-d "${body}" \
-w $'\nHTTP_STATUS=%{http_code}'
else
curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}'
fi
}
if [[ -z "${ACTION_CODE}" ]]; then
usage
exit 1
fi
if [[ "${OPERATION}" != "preview" && "${OPERATION}" != "execute" ]]; then
echo "operation must be preview or execute" >&2
usage
exit 1
fi
ACTION_PAYLOAD_JSON="${RAW_ACTION_PAYLOAD}"
if [[ "${ACTION_PAYLOAD_JSON}" == @* ]]; then
ACTION_PAYLOAD_PATH="${ACTION_PAYLOAD_JSON#@}"
if [[ ! -f "${ACTION_PAYLOAD_PATH}" ]]; then
echo "payload file not found: ${ACTION_PAYLOAD_PATH}" >&2
exit 1
fi
ACTION_PAYLOAD_JSON="$(cat "${ACTION_PAYLOAD_PATH}")"
fi
if [[ -z "${NODE_CODES_INPUT}" ]]; then
NODE_CODES_INPUT="$(python3 - <<'PY' "${BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}"
import json
import sys
import urllib.request
def normalize_codes(*groups):
result = []
for group in groups:
for item in list(group or []):
node_code = str(item or "").strip()
if node_code and node_code not in result:
result.append(node_code)
return result
base_url = str(sys.argv[1] or "").rstrip("/")
action_code = str(sys.argv[2] or "").strip()
payload_raw = str(sys.argv[3] or "").strip() or "{}"
try:
payload = json.loads(payload_raw)
except Exception:
payload = {}
if not isinstance(payload, dict):
payload = {}
payload_node_codes = normalize_codes(payload.get("node_codes") or [])
if payload_node_codes:
print(",".join(payload_node_codes))
raise SystemExit(0)
supported_actions = {
"open_worker_logs_participating",
"run_inspection_participating",
"open_worker_logs_standby",
"run_inspection_standby",
"open_worker_logs",
"run_standard_inspection",
}
if action_code not in supported_actions or not base_url:
print("")
raise SystemExit(0)
try:
with urllib.request.urlopen(f"{base_url}/api/v1/ops/overview", timeout=20) as response:
overview_payload = json.loads(response.read().decode("utf-8"))
overview = dict((overview_payload or {}).get("data") or {})
except Exception:
print("")
raise SystemExit(0)
execution_scene = dict(overview.get("execution_scene") or {})
participation_summary = dict(execution_scene.get("participation_summary") or {})
log_sync = dict(execution_scene.get("log_sync") or {})
dispatch_active = normalize_codes(participation_summary.get("dispatch_active_node_codes") or [])
recent_only = normalize_codes(participation_summary.get("recent_only_node_codes") or [])
standby = normalize_codes(
participation_summary.get("standby_node_codes") or [],
participation_summary.get("non_participating_node_codes") or [],
)
missing_logs = normalize_codes(log_sync.get("missing_participating_nodes") or [])
resolved_node_codes = []
if action_code in {"open_worker_logs_participating", "run_inspection_participating"}:
resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only)
elif action_code in {"open_worker_logs_standby", "run_inspection_standby"}:
resolved_node_codes = normalize_codes(standby)
elif action_code == "open_worker_logs":
resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only, standby)
elif action_code == "run_standard_inspection":
resolved_node_codes = normalize_codes(dispatch_active, recent_only, standby)
print(",".join(resolved_node_codes))
PY
)"
fi
REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}"
import json
import sys
action_code = str(sys.argv[1] or "").strip()
payload_raw = str(sys.argv[2] or "").strip() or "{}"
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action"
node_codes_raw = str(sys.argv[4] or "")
try:
action_payload = json.loads(payload_raw)
except Exception as exc: # pragma: no cover - cli validation
raise SystemExit(f"invalid action payload json: {exc}")
if not isinstance(action_payload, dict):
raise SystemExit("action payload must be a json object")
node_codes = []
seen = set()
for item in node_codes_raw.replace("\n", ",").split(","):
value = str(item or "").strip()
if not value or value in seen:
continue
seen.add(value)
node_codes.append(value)
request_payload = {
"action_code": action_code,
"node_codes": node_codes,
"action_payload": action_payload,
"requested_by": requested_by,
}
print(json.dumps(request_payload, ensure_ascii=False))
PY
)"
PREVIEW_REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}" "${SOURCE_LABEL}" "${ACTION_TITLE}" "${ACTION_SUMMARY}" "${ACTION_REASON}" "${ACTION_META_TEXT}" "${PRIMARY_LABEL}"
import json
import sys
action_code = str(sys.argv[1] or "").strip()
payload_raw = str(sys.argv[2] or "").strip() or "{}"
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action"
node_codes_raw = str(sys.argv[4] or "")
source_label = str(sys.argv[5] or "").strip() or "cli/drive_ops_action"
action_title = str(sys.argv[6] or "").strip()
action_summary = str(sys.argv[7] or "").strip()
action_reason = str(sys.argv[8] or "").strip()
action_meta_text = str(sys.argv[9] or "").strip()
primary_label = str(sys.argv[10] or "").strip()
try:
action_payload = json.loads(payload_raw)
except Exception as exc:
raise SystemExit(f"invalid action payload json: {exc}")
if not isinstance(action_payload, dict):
raise SystemExit("action payload must be a json object")
node_codes = []
seen = set()
for item in node_codes_raw.replace("\n", ",").split(","):
value = str(item or "").strip()
if not value or value in seen:
continue
seen.add(value)
node_codes.append(value)
request_payload = {
"source_label": source_label,
"title": action_title or primary_label or action_code,
"summary": action_summary,
"reason": action_reason,
"meta_text": action_meta_text,
"executor_kind": "driver_action",
"primary_label": primary_label or action_code,
"primary_action_code": action_code,
"primary_node_codes": node_codes,
"primary_action_payload": action_payload,
"requested_by": requested_by,
}
print(json.dumps(request_payload, ensure_ascii=False))
PY
)"
if [[ "${OPERATION}" == "preview" ]]; then
echo "[1/3] driver action preview request"
print_json_or_raw "${PREVIEW_REQUEST_JSON}"
echo
echo
RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/preview" "${PREVIEW_REQUEST_JSON}")"
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
echo "[2/3] raw response"
printf '%s\n' "http_status=${HTTP_STATUS}"
print_json_or_raw "${RESPONSE_JSON}"
echo
echo
echo "[3/3] condensed summary"
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
import json
import sys
payload = json.loads(sys.argv[1])
http_status = int(sys.argv[2] or 0)
data = payload.get("data") or {}
primary = data.get("primary") or {}
secondary = data.get("secondary") or {}
raw_code = payload.get("code", 1)
if raw_code in (0, "0", False):
normalized_code = 0
else:
normalized_code = int(raw_code or 1)
summary = {
"http_status": http_status,
"code": normalized_code,
"message": str(payload.get("message") or ""),
"contract_key": str(data.get("contract_key") or ""),
"contract_version": str(data.get("contract_version") or ""),
"preview_endpoint": str(data.get("preview_endpoint") or ""),
"source_label": str(data.get("source_label") or ""),
"title": str(data.get("title") or ""),
"executor_kind": str(data.get("executor_kind") or ""),
"execution_chain": str(data.get("execution_chain") or ""),
"primary": {
"label": str(primary.get("label") or ""),
"action_code": str(primary.get("action_code") or ""),
"target_method": str(primary.get("target_method") or ""),
"target_api": str(primary.get("target_api") or ""),
"recommendation_label": str(primary.get("recommendation_label") or ""),
"automation_level_label": str(primary.get("automation_level_label") or ""),
"risk_level_label": str(primary.get("risk_level_label") or ""),
"node_codes": list(primary.get("node_codes") or []),
"request_payload": primary.get("request_payload") or {},
},
}
if secondary:
summary["secondary"] = {
"label": str(secondary.get("label") or ""),
"action_code": str(secondary.get("action_code") or ""),
"target_method": str(secondary.get("target_method") or ""),
"target_api": str(secondary.get("target_api") or ""),
"recommendation_label": str(secondary.get("recommendation_label") or ""),
"automation_level_label": str(secondary.get("automation_level_label") or ""),
"risk_level_label": str(secondary.get("risk_level_label") or ""),
"node_codes": list(secondary.get("node_codes") or []),
"request_payload": secondary.get("request_payload") or {},
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
else
echo "[1/3] driver action request"
print_json_or_raw "${REQUEST_JSON}"
echo
echo
RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/execute" "${REQUEST_JSON}")"
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
echo "[2/3] raw response"
printf '%s\n' "http_status=${HTTP_STATUS}"
print_json_or_raw "${RESPONSE_JSON}"
echo
echo
echo "[3/3] condensed summary"
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
import json
import sys
payload = json.loads(sys.argv[1])
http_status = int(sys.argv[2] or 0)
data = payload.get("data") or {}
smart_release_result = data.get("smart_release_result") or {}
ui_intent = data.get("ui_intent") or {}
runtime_settings = data.get("runtime_settings") or {}
raw_code = payload.get("code", 1)
if raw_code in (0, "0", False):
normalized_code = 0
else:
normalized_code = int(raw_code or 1)
summary = {
"http_status": http_status,
"code": normalized_code,
"message": str(payload.get("message") or ""),
"handled": bool(data.get("handled", False)),
"action_code": str(data.get("action_code") or ""),
"mode": str(data.get("mode") or ""),
}
if smart_release_result:
release = smart_release_result.get("release") or {}
rollout = smart_release_result.get("rollout") or {}
smart_plan = smart_release_result.get("smart_rollout") or {}
preview = smart_release_result.get("preview") or {}
summary["smart_release_result"] = {
"release_id": int(release.get("id") or 0),
"release_version": str(release.get("release_version") or ""),
"release_created": bool(smart_release_result.get("release_created", False)),
"release_deduplicated": bool(smart_release_result.get("release_deduplicated", False)),
"rollout_created": bool(smart_release_result.get("rollout_created", False)),
"requires_confirmation": bool(smart_release_result.get("requires_confirmation", False)),
"blocked": bool(smart_release_result.get("blocked", False)),
"rollout_id": int(rollout.get("id") or 0),
"rollout_code": str(rollout.get("rollout_code") or ""),
"smart_mode": str(smart_plan.get("mode") or ""),
"smart_available": bool(smart_plan.get("available", False)),
"target_node_codes": list((smart_plan.get("target_selector") or {}).get("node_codes") or []),
"preview_warnings": list(preview.get("warnings") or []),
"preview_approval_required": bool(preview.get("approval_required", False)),
}
if ui_intent:
summary["ui_intent"] = {
"kind": str(ui_intent.get("kind") or ""),
"release_id": int(ui_intent.get("release_id") or 0),
"template_key": str(ui_intent.get("template_key") or ""),
"playbook_key": str(ui_intent.get("playbook_key") or ""),
"target_node_codes": list(ui_intent.get("target_node_codes") or []),
}
if runtime_settings:
summary["runtime_settings"] = {
"worker_log_sync_enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
"worker_log_sync_mode": str(runtime_settings.get("worker_log_sync_mode") or ""),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
fi
if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then
exit 1
fi

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,476 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib_ops_center.sh"
ops_center_load_config "${SCRIPT_DIR}"
usage() {
cat <<'EOF'
usage:
bash domain-api/deploy/multi-region/drive_release_hub.sh <base_url> <command> [args...]
commands:
launchpad
latest-package
latest-release
preview-latest
preview-latest-smart <worker|control>
preview-release <release_id> <worker|control>
create-latest-release
list-releases [limit]
list-rollouts <release_id> [limit]
smart-latest <worker|control> [confirm]
smart-release <release_id> <worker|control> [confirm]
examples:
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 latest-package
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest-smart worker
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 create-latest-release
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-latest worker
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-release 18 control
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-release 18 control confirm
EOF
}
BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}"
COMMAND="${2:-}"
ARG1="${3:-}"
ARG2="${4:-}"
ARG3="${5:-}"
REQUESTED_BY="${REQUESTED_BY:-$(ops_center_resolve_requested_by)}"
print_json_or_raw() {
local raw_text="${1:-}"
python3 - "${raw_text}" <<'PY'
import json
import sys
raw_text = str(sys.argv[1] or "")
try:
payload = json.loads(raw_text)
except Exception:
print(raw_text)
else:
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY
}
request_with_status() {
local method="$1"
local url="$2"
local body="${3:-}"
if [[ "${method}" == "POST" ]]; then
curl -sS -X POST "${url}" \
-H 'Content-Type: application/json' \
-d "${body}" \
-w $'\nHTTP_STATUS=%{http_code}'
else
curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}'
fi
}
if [[ -z "${COMMAND}" ]]; then
usage
exit 1
fi
METHOD="GET"
PATH_SUFFIX=""
REQUEST_JSON=""
confirm_to_bool() {
local raw_value="${1:-}"
case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in
1|true|yes|y|confirm|confirmed)
printf 'true'
;;
*)
printf 'false'
;;
esac
}
case "${COMMAND}" in
launchpad)
CHANNEL="${ARG1:-stable}"
PATH_SUFFIX="/api/v1/ops/releases/launchpad?channel=${CHANNEL}"
;;
latest-package)
PATH_SUFFIX="/api/v1/ops/releases/package-metadata/latest"
;;
latest-release)
PATH_SUFFIX="/api/v1/ops/releases/latest"
;;
preview-latest)
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/preview"
REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}"
import json
import sys
requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub"
payload = {
"created_by": f"{requested_by}/latest-package-preview",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
preview-latest-smart)
MODE="${ARG1:-}"
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
echo "mode must be worker or control" >&2
usage
exit 1
fi
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout-preview"
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}"
import json
import sys
mode = str(sys.argv[1] or "").strip()
requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub"
payload = {
"mode": mode,
"created_by": f"{requested_by}/latest-package-preview",
"rollout_created_by": f"{requested_by}/latest-package-preview/rollout",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
create-latest-release)
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest"
REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}"
import json
import sys
requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub"
payload = {
"created_by": f"{requested_by}/latest-package",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
list-releases)
RELEASE_LIMIT="${ARG1:-10}"
PATH_SUFFIX="/api/v1/ops/releases?limit=${RELEASE_LIMIT}"
;;
list-rollouts)
RELEASE_ID="${ARG1:-}"
ROLLOUT_LIMIT="${ARG2:-10}"
if [[ -z "${RELEASE_ID}" ]]; then
echo "release_id is required for list-rollouts" >&2
usage
exit 1
fi
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/rollouts?limit=${ROLLOUT_LIMIT}"
;;
smart-latest)
MODE="${ARG1:-}"
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
echo "mode must be worker or control" >&2
usage
exit 1
fi
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout"
CONFIRM_RISKY="$(confirm_to_bool "${ARG2:-}")"
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}"
import json
import sys
mode = str(sys.argv[1] or "").strip()
confirm_risky = str(sys.argv[2] or "").strip().lower() == "true"
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub"
payload = {
"mode": mode,
"confirm_risky": confirm_risky,
"created_by": f"{requested_by}/latest-package",
"rollout_created_by": f"{requested_by}/latest-package/rollout",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
preview-release)
RELEASE_ID="${ARG1:-}"
MODE="${ARG2:-}"
if [[ -z "${RELEASE_ID}" ]]; then
echo "release_id is required for preview-release" >&2
usage
exit 1
fi
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
echo "mode must be worker or control" >&2
usage
exit 1
fi
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout-preview"
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}"
import json
import sys
mode = str(sys.argv[1] or "").strip()
requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub"
payload = {
"mode": mode,
"created_by": f"{requested_by}/existing-release-preview",
"rollout_created_by": f"{requested_by}/existing-release-preview/rollout",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
smart-release)
RELEASE_ID="${ARG1:-}"
MODE="${ARG2:-}"
if [[ -z "${RELEASE_ID}" ]]; then
echo "release_id is required for smart-release" >&2
usage
exit 1
fi
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
echo "mode must be worker or control" >&2
usage
exit 1
fi
METHOD="POST"
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout"
CONFIRM_RISKY="$(confirm_to_bool "${ARG3:-}")"
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}"
import json
import sys
mode = str(sys.argv[1] or "").strip()
confirm_risky = str(sys.argv[2] or "").strip().lower() == "true"
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub"
payload = {
"mode": mode,
"confirm_risky": confirm_risky,
"created_by": f"{requested_by}/existing-release",
"rollout_created_by": f"{requested_by}/existing-release/rollout",
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
;;
*)
echo "unknown command: ${COMMAND}" >&2
usage
exit 1
;;
esac
echo "[1/3] request"
printf '%s\n' "method=${METHOD}"
printf '%s\n' "url=${BASE_URL}${PATH_SUFFIX}"
if [[ -n "${REQUEST_JSON}" ]]; then
print_json_or_raw "${REQUEST_JSON}"
else
echo "(no request body)"
fi
echo
echo
RAW_HTTP_RESPONSE="$(request_with_status "${METHOD}" "${BASE_URL}${PATH_SUFFIX}" "${REQUEST_JSON}")"
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
echo "[2/3] raw response"
printf '%s\n' "http_status=${HTTP_STATUS}"
print_json_or_raw "${RESPONSE_JSON}"
echo
echo
echo "[3/3] condensed summary"
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
import json
import sys
raw_response = str(sys.argv[1] or "")
http_status = int(sys.argv[2] or 0)
try:
payload = json.loads(raw_response)
except Exception:
print(json.dumps({
"http_status": http_status,
"code": 1,
"message": "response is not valid json",
"raw_response": raw_response[:500],
}, ensure_ascii=False, indent=2))
sys.exit(0 if 200 <= http_status < 300 else 1)
data = payload.get("data") or {}
raw_code = payload.get("code", 1)
if raw_code in (0, "0", False):
normalized_code = 0
else:
normalized_code = int(raw_code or 1)
summary = {
"http_status": http_status,
"code": normalized_code,
"message": str(payload.get("message") or ""),
}
if isinstance(data, dict) and "package" in data:
package = dict(data.get("package") or {})
summary["package"] = {
"available": bool(package.get("available", False)),
"package_name": str(package.get("package_name") or ""),
"package_type": str(package.get("package_type") or ""),
"generated_at": str(package.get("generated_at") or ""),
"smoke_test_ok": bool(package.get("smoke_test_ok", False)),
"smoke_test_message": str(package.get("smoke_test_message") or ""),
}
if isinstance(data, dict) and "launchpad_status" in data:
launchpad_status = dict(data.get("launchpad_status") or {})
worker_preview = dict(data.get("worker_rollout_preview") or {})
worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {})
control_preview = dict(data.get("control_rollout_preview") or {})
control_smart_rollout = dict(control_preview.get("smart_rollout") or {})
summary["launchpad"] = {
"channel": str(data.get("channel") or ""),
"status": str(launchpad_status.get("status") or ""),
"status_label": str(launchpad_status.get("status_label") or ""),
"summary": str(launchpad_status.get("summary") or ""),
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
"recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""),
"recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""),
"recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""),
"onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
"onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
"recommended_command": str(launchpad_status.get("recommended_command") or ""),
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
"recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""),
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
"latest_package": dict(data.get("latest_package") or {}),
"latest_release": {
"id": int(((data.get("latest_release") or {}).get("id") or 0)),
"release_version": str((data.get("latest_release") or {}).get("release_version") or ""),
"status": str((data.get("latest_release") or {}).get("status") or ""),
},
"worker_rollout_preview": {
"message": str(worker_preview.get("message") or ""),
"available": bool(worker_smart_rollout.get("available", False)),
"requires_confirmation": bool(worker_preview.get("requires_confirmation", False)),
"blocked": bool(worker_preview.get("blocked", False)),
"execution_mode": str(worker_smart_rollout.get("execution_mode") or ""),
"execution_mode_label": str(worker_smart_rollout.get("execution_mode_label") or ""),
"target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []),
},
"control_rollout_preview": {
"message": str(control_preview.get("message") or ""),
"available": bool(control_smart_rollout.get("available", False)),
"requires_confirmation": bool(control_preview.get("requires_confirmation", False)),
"blocked": bool(control_preview.get("blocked", False)),
"execution_mode": str(control_smart_rollout.get("execution_mode") or ""),
"execution_mode_label": str(control_smart_rollout.get("execution_mode_label") or ""),
"target_node_codes": list(control_smart_rollout.get("target_node_codes") or []),
},
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
sys.exit(0)
if isinstance(data, dict) and "available" in data:
summary["latest_package"] = {
"available": bool(data.get("available", False)),
"package_name": str(data.get("package_name") or ""),
"package_type": str(data.get("package_type") or ""),
"generated_at": str(data.get("generated_at") or ""),
"commit_sha": str(data.get("commit_sha") or ""),
"smoke_test_ok": bool(data.get("smoke_test_ok", False)),
"smoke_test_message": str(data.get("smoke_test_message") or ""),
"reason": str(data.get("reason") or ""),
}
elif isinstance(data, dict) and "releases" in data:
releases = list(data.get("releases") or [])
summary["releases"] = {
"count": len(releases),
"items": [
{
"id": int(item.get("id") or 0),
"release_version": str(item.get("release_version") or ""),
"channel": str(item.get("channel") or ""),
"status": str(item.get("status") or ""),
}
for item in releases[:5]
],
}
elif isinstance(data, dict) and "rollouts" in data:
rollouts = list(data.get("rollouts") or [])
summary["rollouts"] = {
"count": len(rollouts),
"items": [
{
"id": int(item.get("id") or 0),
"rollout_code": str(item.get("rollout_code") or ""),
"status": str(item.get("status") or ""),
"jobs_total": int(item.get("jobs_total") or 0),
"jobs_created": int(item.get("jobs_created") or 0),
}
for item in rollouts[:5]
],
}
else:
release = data.get("release") or data
rollout = data.get("rollout") or {}
smart_rollout = data.get("smart_rollout") or {}
preview = data.get("preview") or {}
summary["preview_source"] = str(data.get("preview_source") or "")
summary["release"] = {
"id": int(release.get("id") or 0),
"release_version": str(release.get("release_version") or ""),
"channel": str(release.get("channel") or ""),
"status": str(release.get("status") or ""),
"is_preview": bool(release.get("is_preview", False)),
}
if rollout:
summary["rollout"] = {
"id": int(rollout.get("id") or 0),
"rollout_code": str(rollout.get("rollout_code") or ""),
"status": str(rollout.get("status") or ""),
}
if smart_rollout:
summary["smart_rollout"] = {
"available": bool(smart_rollout.get("available", False)),
"mode": str(smart_rollout.get("mode") or ""),
"mode_label": str(smart_rollout.get("mode_label") or ""),
"target_node_codes": list((smart_rollout.get("target_selector") or {}).get("node_codes") or []),
"summary": str(smart_rollout.get("summary") or smart_rollout.get("reason") or ""),
}
if preview:
summary["preview"] = {
"blocked": bool(preview.get("blocked", False)),
"approval_required": bool(preview.get("approval_required", False)),
"warnings": list(preview.get("warnings") or []),
}
summary["result_flags"] = {
"release_created": bool(data.get("release_created", False)),
"release_deduplicated": bool(data.get("release_deduplicated", False)),
"rollout_created": bool(data.get("rollout_created", False)),
"requires_confirmation": bool(data.get("requires_confirmation", False)),
"blocked": bool(data.get("blocked", False)),
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then
exit 1
fi

View File

@@ -0,0 +1,556 @@
#!/usr/bin/env bash
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/lib_ops_center.sh"
ops_center_load_config "${SCRIPT_DIR}"
MAINLAND_BASE_URL_DEFAULT="$(ops_center_resolve_mainland_api_base_url)"
OVERSEAS_BASE_URL_DEFAULT="$(ops_center_resolve_overseas_api_base_url)"
REPORT_ROOT_DEFAULT="${OPS_CENTER_REPORT_ROOT}/go-live-bundles"
REPORT_DIR_INPUT="${1:-}"
MAINLAND_BASE_URL_INPUT="${2:-}"
OVERSEAS_BASE_URL_INPUT="${3:-}"
timestamp_now() {
date '+%Y%m%d_%H%M%S'
}
normalize_optional_base_url() {
local raw_value="${1:-}"
local fallback_value="${2:-}"
if [[ -z "${raw_value}" ]]; then
printf '%s' "${fallback_value}"
return
fi
case "${raw_value}" in
-|none|null|disabled)
printf ''
;;
*)
printf '%s' "${raw_value}"
;;
esac
}
if [[ -n "${REPORT_DIR_INPUT}" && "${REPORT_DIR_INPUT}" != http://* && "${REPORT_DIR_INPUT}" != https://* ]]; then
REPORT_DIR="${REPORT_DIR_INPUT}"
MAINLAND_BASE_URL="${MAINLAND_BASE_URL_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}"
OVERSEAS_BASE_URL="$(normalize_optional_base_url "${OVERSEAS_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")"
else
REPORT_DIR="${REPORT_ROOT_DEFAULT}/go-live-bundle-$(timestamp_now)"
MAINLAND_BASE_URL="${REPORT_DIR_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}"
OVERSEAS_BASE_URL="$(normalize_optional_base_url "${MAINLAND_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")"
fi
mkdir -p "${REPORT_DIR}"
CURL_CONNECT_TIMEOUT="${OPS_CENTER_CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${OPS_CENTER_CURL_MAX_TIME:-15}"
STEP_COMMAND_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS:-45}"
DOCTOR_DECISION_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_DECISION_TIMEOUT_SECONDS:-120}"
DOCTOR_EXPORT_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_EXPORT_TIMEOUT_SECONDS:-180}"
DOCTOR_INNER_REPORT_TIMEOUT_SECONDS="${OPS_CENTER_DOCTOR_INNER_REPORT_TIMEOUT_SECONDS:-60}"
ENV_AUDIT_TXT_PATH="${REPORT_DIR}/00_env_audit.txt"
SUMMARY_TXT_PATH="${REPORT_DIR}/01_go_live_check_summary.txt"
GO_LIVE_JSON_PATH="${REPORT_DIR}/02_go_live_summary.json"
STACK_JSON_PATH="${REPORT_DIR}/03_stack_diagnosis.json"
DRIVER_FEED_JSON_PATH="${REPORT_DIR}/04_driver_feed.json"
CODEX_BRIEF_JSON_PATH="${REPORT_DIR}/05_codex_brief.json"
RELEASE_LAUNCHPAD_JSON_PATH="${REPORT_DIR}/06_release_launchpad.json"
DOCTOR_DECISION_JSON_PATH="${REPORT_DIR}/07_doctor_decision.json"
DOCTOR_EXPORT_STDOUT_PATH="${REPORT_DIR}/08_doctor_export_stdout.txt"
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
declare -a STEP_RESULTS=()
pretty_print_json_file() {
local source_path="${1:-}"
python3 - "${source_path}" <<'PY'
import json
import pathlib
import sys
path = pathlib.Path(sys.argv[1])
raw_text = path.read_text(encoding="utf-8") if path.exists() else ""
try:
payload = json.loads(raw_text)
except Exception:
print(raw_text)
else:
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY
}
record_step_result() {
local key="${1:-}"
local path="${2:-}"
local ok="${3:-false}"
local http_status="${4:-}"
local exit_code="${5:-0}"
STEP_RESULTS+=("${key}|${path}|${ok}|${http_status}|${exit_code}")
}
fetch_json_to_file() {
local key="${1:-}"
local url="${2:-}"
local output_path="${3:-}"
local tmp_path
tmp_path="$(mktemp)"
local raw_text http_status
raw_text="$(curl -sS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"${url}" \
-w $'\nHTTP_STATUS=%{http_code}' || true)"
http_status="${raw_text##*$'\n'HTTP_STATUS=}"
printf '%s' "${raw_text%$'\n'HTTP_STATUS=*}" > "${tmp_path}"
pretty_print_json_file "${tmp_path}" > "${output_path}"
rm -f "${tmp_path}"
if [[ "${http_status}" == 2* ]]; then
record_step_result "${key}" "${output_path}" "true" "${http_status}" "0"
return 0
fi
record_step_result "${key}" "${output_path}" "false" "${http_status}" "1"
return 1
}
capture_command_to_file() {
local key="${1:-}"
local output_path="${2:-}"
shift 2 || true
capture_command_to_file_with_timeout "${STEP_COMMAND_TIMEOUT_SECONDS}" "${key}" "${output_path}" "$@"
}
capture_command_to_file_with_timeout() {
local timeout_seconds="${1:-45}"
local key="${2:-}"
local output_path="${3:-}"
shift 3 || true
local -a wrapped_command=("$@")
local had_errexit="false"
case "$-" in
*e*) had_errexit="true" ;;
esac
if command -v timeout >/dev/null 2>&1; then
wrapped_command=(timeout "${timeout_seconds}" "$@")
fi
set +e
"${wrapped_command[@]}" > "${output_path}" 2>&1
local exit_code=$?
if [[ "${had_errexit}" == "true" ]]; then
set -e
else
set +e
fi
if [[ "${exit_code}" -eq 0 ]]; then
record_step_result "${key}" "${output_path}" "true" "" "${exit_code}"
return 0
fi
record_step_result "${key}" "${output_path}" "false" "" "${exit_code}"
return "${exit_code}"
}
materialize_go_live_summary_fallback() {
local summary_path="${1:-}"
local output_path="${2:-}"
python3 - "${summary_path}" "${output_path}" <<'PY'
import json
import pathlib
import sys
summary_path = pathlib.Path(sys.argv[1])
output_path = pathlib.Path(sys.argv[2])
raw_text = summary_path.read_text(encoding="utf-8") if summary_path.exists() else ""
start = raw_text.find("{")
end = raw_text.rfind("}")
if start < 0 or end < start:
sys.exit(1)
json_text = raw_text[start:end + 1]
payload = json.loads(json_text)
if not isinstance(payload, dict):
sys.exit(1)
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
PY
}
override_step_result() {
local key="${1:-}"
local path="${2:-}"
local ok="${3:-false}"
local http_status="${4:-}"
local exit_code="${5:-0}"
local -a updated_results=()
local raw_item existing_key
for raw_item in "${STEP_RESULTS[@]}"; do
existing_key="${raw_item%%|*}"
if [[ "${existing_key}" == "${key}" ]]; then
continue
fi
updated_results+=("${raw_item}")
done
STEP_RESULTS=("${updated_results[@]}")
record_step_result "${key}" "${path}" "${ok}" "${http_status}" "${exit_code}"
}
set +e
capture_command_to_file \
"env_audit" \
"${ENV_AUDIT_TXT_PATH}" \
bash "${SCRIPT_DIR}/drive_ops_center.sh" env-audit "${MAINLAND_BASE_URL}"
capture_command_to_file \
"go_live_check_summary" \
"${SUMMARY_TXT_PATH}" \
bash "${SCRIPT_DIR}/drive_ops_center.sh" go-live-check "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" summary
fetch_json_to_file \
"go_live_summary" \
"${MAINLAND_BASE_URL}/api/v1/ops/go-live-summary?base_url=${MAINLAND_BASE_URL}" \
"${GO_LIVE_JSON_PATH}"
if [[ ! -s "${GO_LIVE_JSON_PATH}" || "$(sed -n '1p' "${GO_LIVE_JSON_PATH}" 2>/dev/null)" != "{"* ]]; then
if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then
override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0"
fi
fi
if [[ "$(python3 - "${GO_LIVE_JSON_PATH}" <<'PY'
import json
import pathlib
import sys
path = pathlib.Path(sys.argv[1])
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception:
print("invalid")
raise SystemExit(0)
if isinstance(payload, dict) and payload.get("detail") == "Not Found":
print("not_found")
else:
print("ok")
PY
)" == "not_found" ]]; then
if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then
override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0"
fi
fi
fetch_json_to_file \
"stack_diagnosis" \
"${MAINLAND_BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${MAINLAND_BASE_URL}" \
"${STACK_JSON_PATH}"
fetch_json_to_file \
"driver_feed" \
"${MAINLAND_BASE_URL}/api/v1/ops/driver-feed" \
"${DRIVER_FEED_JSON_PATH}"
fetch_json_to_file \
"codex_brief" \
"${MAINLAND_BASE_URL}/api/v1/ops/codex-brief" \
"${CODEX_BRIEF_JSON_PATH}"
fetch_json_to_file \
"release_launchpad" \
"${MAINLAND_BASE_URL}/api/v1/ops/releases/launchpad" \
"${RELEASE_LAUNCHPAD_JSON_PATH}"
capture_command_to_file_with_timeout \
"${DOCTOR_EXPORT_TIMEOUT_SECONDS}" \
"doctor_export" \
"${DOCTOR_EXPORT_STDOUT_PATH}" \
env "OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS=${DOCTOR_INNER_REPORT_TIMEOUT_SECONDS}" \
bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-export "${REPORT_DIR}/doctor-export" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}"
capture_command_to_file_with_timeout \
"${DOCTOR_DECISION_TIMEOUT_SECONDS}" \
"doctor_decision" \
"${DOCTOR_DECISION_JSON_PATH}" \
bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${REPORT_DIR}/doctor-export"
set -e
python3 - <<'PY' \
"${MANIFEST_PATH}" \
"${REPORT_DIR}" \
"${MAINLAND_BASE_URL}" \
"${OVERSEAS_BASE_URL}" \
"${ENV_AUDIT_TXT_PATH}" \
"${SUMMARY_TXT_PATH}" \
"${GO_LIVE_JSON_PATH}" \
"${STACK_JSON_PATH}" \
"${DRIVER_FEED_JSON_PATH}" \
"${CODEX_BRIEF_JSON_PATH}" \
"${RELEASE_LAUNCHPAD_JSON_PATH}" \
"${DOCTOR_DECISION_JSON_PATH}" \
"${DOCTOR_EXPORT_STDOUT_PATH}" \
"${STEP_RESULTS[@]}"
import json
import pathlib
import sys
manifest_path = pathlib.Path(sys.argv[1])
report_dir = pathlib.Path(sys.argv[2])
mainland_base_url = sys.argv[3]
overseas_base_url = sys.argv[4]
file_keys = [
"env_audit",
"go_live_check_summary",
"go_live_summary",
"stack_diagnosis",
"driver_feed",
"codex_brief",
"release_launchpad",
"doctor_decision",
"doctor_export",
]
file_paths = {
"env_audit": pathlib.Path(sys.argv[5]),
"go_live_check_summary": pathlib.Path(sys.argv[6]),
"go_live_summary": pathlib.Path(sys.argv[7]),
"stack_diagnosis": pathlib.Path(sys.argv[8]),
"driver_feed": pathlib.Path(sys.argv[9]),
"codex_brief": pathlib.Path(sys.argv[10]),
"release_launchpad": pathlib.Path(sys.argv[11]),
"doctor_decision": pathlib.Path(sys.argv[12]),
"doctor_export": pathlib.Path(sys.argv[13]),
}
step_results = {}
for raw_item in sys.argv[14:]:
parts = raw_item.split("|", 4)
if len(parts) != 5:
continue
key, path_value, ok_value, http_status, exit_code = parts
step_results[key] = {
"path": path_value,
"ok": ok_value == "true",
"http_status": http_status,
"exit_code": int(exit_code or 0),
}
def load_json_file(path: pathlib.Path) -> dict:
if not path.is_file():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {}
return payload if isinstance(payload, dict) else {}
def load_env_audit_summary(path: pathlib.Path) -> dict:
if not path.is_file():
return {}
raw_text = path.read_text(encoding="utf-8", errors="replace")
marker = "[8/8] condensed env audit summary"
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
if not candidate_text:
return {}
try:
payload = json.loads(candidate_text)
except Exception:
return {}
return payload if isinstance(payload, dict) else {}
failures = []
artifacts = []
for key in file_keys:
artifact_path = file_paths[key]
result = dict(step_results.get(key) or {})
if not result:
result = {
"path": str(artifact_path),
"ok": artifact_path.exists(),
"http_status": "",
"exit_code": 0 if artifact_path.exists() else 1,
}
entry = {
"key": key,
"path": str(artifact_path),
"exists": artifact_path.exists(),
"ok": bool(result.get("ok", False)),
"http_status": str(result.get("http_status") or ""),
"exit_code": int(result.get("exit_code", 0) or 0),
}
artifacts.append(entry)
if not entry["ok"]:
failures.append(key)
go_live_summary_payload = load_json_file(file_paths["go_live_summary"])
stack_diagnosis_payload = load_json_file(file_paths["stack_diagnosis"])
driver_feed_payload = load_json_file(file_paths["driver_feed"])
codex_brief_payload = load_json_file(file_paths["codex_brief"])
env_audit_summary = load_env_audit_summary(file_paths["env_audit"])
stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {})
driver_feed_summary = dict(driver_feed_payload.get("summary") or {})
codex_brief_summary = dict(codex_brief_payload.get("summary") or {})
def first_nonempty(*values: object) -> str:
for value in values:
normalized = str(value or "").strip()
if normalized:
return normalized
return ""
def nonempty_distinct(values: dict[str, str]) -> list[str]:
return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()})
def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]:
return sorted({int(values[source]) for source in values if source in available_sources})
launchpad_target_node_codes = {
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip(),
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
}
launchpad_recovery_labels = {
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip(),
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
}
launchpad_recovery_summaries = {
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip(),
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
}
launchpad_bootstrap_pending_nodes = {
"go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
}
launchpad_acceptance_ready_nodes = {
"go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
}
available_alignment_sources = {
source
for source, payload_value in {
"go_live_summary": go_live_summary_payload,
"stack_diagnosis": stack_diagnosis_payload,
"driver_feed": driver_feed_payload,
"codex_brief": codex_brief_payload,
}.items()
if isinstance(payload_value, dict) and payload_value
}
launchpad_alignment = {
"consistent": True,
"target_node_code_consistent": True,
"recovery_label_consistent": True,
"recovery_summary_consistent": True,
"bootstrap_pending_consistent": True,
"acceptance_ready_consistent": True,
"available_sources": sorted(available_alignment_sources),
"target_node_codes": launchpad_target_node_codes,
"recovery_labels": launchpad_recovery_labels,
"recovery_summaries": launchpad_recovery_summaries,
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
}
launchpad_alignment["target_node_code_consistent"] = len(nonempty_distinct({
source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources
})) <= 1
launchpad_alignment["recovery_label_consistent"] = len(nonempty_distinct({
source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources
})) <= 1
launchpad_alignment["recovery_summary_consistent"] = len(nonempty_distinct({
source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources
})) <= 1
launchpad_alignment["bootstrap_pending_consistent"] = len(
distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources)
) <= 1
launchpad_alignment["acceptance_ready_consistent"] = len(
distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources)
) <= 1
launchpad_alignment["consistent"] = all([
bool(launchpad_alignment["target_node_code_consistent"]),
bool(launchpad_alignment["recovery_label_consistent"]),
bool(launchpad_alignment["recovery_summary_consistent"]),
bool(launchpad_alignment["bootstrap_pending_consistent"]),
bool(launchpad_alignment["acceptance_ready_consistent"]),
])
env_audit_status = str(env_audit_summary.get("status") or "").strip()
env_audit_runtime_may_need_restart = bool(((env_audit_summary.get("runtime") or {}).get("runtime_may_need_restart", False)))
review_status_hint = "ready"
review_headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
if failures or env_audit_status == "blocked":
review_status_hint = "blocked"
review_headline = "当前导出包仍有关键阻断项,不能直接作为正式上线终稿。"
elif env_audit_status == "attention" or not launchpad_alignment["consistent"]:
review_status_hint = "attention"
if not launchpad_alignment["consistent"]:
review_headline = "核心报告虽已导出,但 launchpad 摘要在不同 surface 之间存在漂移,建议先重新复核。"
else:
review_headline = "核心报告已齐,但环境审计仍提示存在待收口事项。"
payload = {
"report_dir": str(report_dir),
"mainland_base_url": mainland_base_url,
"overseas_base_url": overseas_base_url,
"artifacts": artifacts,
"failures": failures,
"summary": {
"ok": len(failures) == 0,
"artifact_total": len(artifacts),
"failure_total": len(failures),
"review_status_hint": review_status_hint,
"review_headline": review_headline,
"env_audit_status": env_audit_status,
"env_audit_runtime_may_need_restart": env_audit_runtime_may_need_restart,
"launchpad_recommended_target_node_code": first_nonempty(
go_live_summary_payload.get("launchpad_recommended_target_node_code"),
stack_diagnosis.get("launchpad_recommended_target_node_code"),
driver_feed_summary.get("launchpad_recommended_target_node_code"),
codex_brief_summary.get("launchpad_recommended_target_node_code"),
),
"launchpad_recommended_recovery_label": first_nonempty(
go_live_summary_payload.get("launchpad_recommended_recovery_label"),
stack_diagnosis.get("launchpad_recommended_recovery_label"),
driver_feed_summary.get("launchpad_recommended_recovery_label"),
codex_brief_summary.get("launchpad_recommended_recovery_label"),
),
"launchpad_recommended_recovery_summary": first_nonempty(
go_live_summary_payload.get("launchpad_recommended_recovery_summary"),
stack_diagnosis.get("launchpad_recommended_recovery_summary"),
driver_feed_summary.get("launchpad_recommended_recovery_summary"),
codex_brief_summary.get("launchpad_recommended_recovery_summary"),
),
"launchpad_onboarding_bootstrap_pending_nodes": max(launchpad_bootstrap_pending_nodes.values() or [0]),
"launchpad_onboarding_acceptance_ready_nodes": max(launchpad_acceptance_ready_nodes.values() or [0]),
"launchpad_alignment": launchpad_alignment,
"recommended_reading_order": [
"00_env_audit.txt",
"01_go_live_check_summary.txt",
"02_go_live_summary.json",
"03_stack_diagnosis.json",
"04_driver_feed.json",
"05_codex_brief.json",
"06_release_launchpad.json",
"07_doctor_decision.json",
"08_doctor_export_stdout.txt",
],
},
}
manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(payload, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,135 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEMPLATE_PATH="${SCRIPT_DIR}/templates/domaincheck-ops-center.env.example"
usage() {
cat <<'EOF'
usage:
bash domain-api/deploy/multi-region/init_ops_center_config.sh \
[config_path] [mainland_api_base_url] [overseas_api_base_url] [control_plane_base_url]
examples:
bash domain-api/deploy/multi-region/init_ops_center_config.sh
bash domain-api/deploy/multi-region/init_ops_center_config.sh /etc/default/domaincheck-ops-center \
http://121.204.244.188:8100 \
http://152.53.37.118:8100 \
https://api.example.com
notes:
- existing config files are preserved unless FORCE=true
- advanced fields can be overridden with env vars:
OPS_ACTIVE_PROFILE
OPS_PROFILE_NAMES
OPS_DEFAULT_REQUESTED_BY
OPS_DEFAULT_ISSUED_BY
OPS_DEFAULT_EXPIRES_IN_HOURS
OPS_DEFAULT_ROOT_DIR
OPS_REPORT_ROOT
EOF
}
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
usage
exit 0
fi
TARGET_PATH="${1:-/etc/default/domaincheck-ops-center}"
MAINLAND_API_BASE_URL="${2:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}"
OVERSEAS_API_BASE_URL="${3:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}"
CONTROL_PLANE_BASE_URL="${4:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}"
ACTIVE_PROFILE="${OPS_ACTIVE_PROFILE:-default}"
PROFILE_NAMES="${OPS_PROFILE_NAMES:-${ACTIVE_PROFILE}}"
DEFAULT_REQUESTED_BY="${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}"
DEFAULT_ISSUED_BY="${OPS_DEFAULT_ISSUED_BY:-ops-center}"
DEFAULT_EXPIRES_IN_HOURS="${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}"
DEFAULT_ROOT_DIR="${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}"
REPORT_ROOT="${OPS_REPORT_ROOT:-}"
NODE_AGENT_SERVICE_NAME="${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
NODE_AGENT_ENV_FILE="${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
FORCE_WRITE="${FORCE:-false}"
if [[ ! -f "${TEMPLATE_PATH}" ]]; then
echo "template not found: ${TEMPLATE_PATH}" >&2
exit 1
fi
mkdir -p "$(dirname "${TARGET_PATH}")"
if [[ ! -f "${TARGET_PATH}" || "${FORCE_WRITE}" == "true" ]]; then
cp "${TEMPLATE_PATH}" "${TARGET_PATH}"
fi
upsert_key() {
local key="$1"
local value="$2"
if grep -q "^${key}=" "${TARGET_PATH}"; then
sed -i "s#^${key}=.*#${key}=${value}#" "${TARGET_PATH}"
else
printf '%s=%s\n' "${key}" "${value}" >>"${TARGET_PATH}"
fi
}
upsert_commented_key() {
local key="$1"
local value="$2"
if [[ -z "${value}" ]]; then
return 0
fi
upsert_key "${key}" "${value}"
}
upsert_key "OPS_ACTIVE_PROFILE" "${ACTIVE_PROFILE}"
upsert_key "OPS_PROFILE_NAMES" "${PROFILE_NAMES}"
upsert_key "OPS_MAINLAND_API_BASE_URL" "${MAINLAND_API_BASE_URL}"
upsert_key "OPS_OVERSEAS_API_BASE_URL" "${OVERSEAS_API_BASE_URL}"
upsert_key "OPS_DEFAULT_CONTROL_PLANE_BASE_URL" "${CONTROL_PLANE_BASE_URL}"
upsert_key "OPS_DEFAULT_REQUESTED_BY" "${DEFAULT_REQUESTED_BY}"
upsert_key "OPS_DEFAULT_ISSUED_BY" "${DEFAULT_ISSUED_BY}"
upsert_key "OPS_DEFAULT_EXPIRES_IN_HOURS" "${DEFAULT_EXPIRES_IN_HOURS}"
upsert_key "OPS_DEFAULT_ROOT_DIR" "${DEFAULT_ROOT_DIR}"
upsert_key "OPS_NODE_AGENT_SERVICE_NAME" "${NODE_AGENT_SERVICE_NAME}"
upsert_key "OPS_NODE_AGENT_ENV_FILE" "${NODE_AGENT_ENV_FILE}"
upsert_commented_key "OPS_REPORT_ROOT" "${REPORT_ROOT}"
python3 - <<'PY' "${TARGET_PATH}"
import json
import sys
from pathlib import Path
config_path = Path(sys.argv[1])
keys = [
"OPS_ACTIVE_PROFILE",
"OPS_PROFILE_NAMES",
"OPS_MAINLAND_API_BASE_URL",
"OPS_OVERSEAS_API_BASE_URL",
"OPS_DEFAULT_CONTROL_PLANE_BASE_URL",
"OPS_DEFAULT_REQUESTED_BY",
"OPS_DEFAULT_ISSUED_BY",
"OPS_DEFAULT_EXPIRES_IN_HOURS",
"OPS_DEFAULT_ROOT_DIR",
"OPS_NODE_AGENT_SERVICE_NAME",
"OPS_NODE_AGENT_ENV_FILE",
"OPS_REPORT_ROOT",
]
values = {}
for line in config_path.read_text(encoding="utf-8").splitlines():
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
if key in keys:
values[key] = value
print(json.dumps({
"ok": True,
"config_path": str(config_path),
"summary": values,
"next_commands": [
"bash domain-api/deploy/multi-region/drive_ops_center.sh config",
"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export",
"bash domain-api/deploy/multi-region/drive_ops_center.sh agent-check",
],
}, ensure_ascii=False, indent=2))
PY

View File

@@ -100,7 +100,7 @@ REDIS_DB=0
EOF
cd "${DOMAIN_API_DIR}"
bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller
bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller
cat >/etc/default/domaincheck-worker <<EOF
WORKER_MODE=linux-systemd
@@ -138,4 +138,4 @@ chown -R www:www "${REPO_DIR}"
echo
echo "国内 controller 部署完成,建议继续执行:"
echo "cd ${DOMAIN_API_DIR} && bash deploy/multi-region/check_mainland_controller.sh"
echo "cd ${DOMAIN_API_DIR} && bash domain-api/deploy/multi-region/check_mainland_controller.sh"

View File

@@ -101,7 +101,7 @@ REDIS_DB=0
EOF
cd "${DOMAIN_API_DIR}"
bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker
bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker
cat >/etc/default/domaincheck-worker <<EOF
WORKER_MODE=linux-systemd

View File

@@ -185,6 +185,15 @@ sed -i 's#^NODE_ROLE=.*#NODE_ROLE=control#' /etc/default/domaincheck-api
sed -i 's#^SYNC_PUSH_ENABLED=.*#SYNC_PUSH_ENABLED=false#' /etc/default/domaincheck-api
sed -i "s#^CORS_ORIGINS=.*#CORS_ORIGINS='[\\\"https://${ADMIN_DOMAIN}\\\",\\\"http://127.0.0.1:3201\\\",\\\"http://localhost:3201\\\"]'#" /etc/default/domaincheck-api
OPS_DEFAULT_REQUESTED_BY="cli/${OVERSEAS_NODE_CODE}" \
OPS_DEFAULT_ISSUED_BY="${OVERSEAS_NODE_CODE}" \
OPS_DEFAULT_ROOT_DIR="${BASE_DIR}" \
bash "${DOMAIN_API_DIR}/deploy/multi-region/init_ops_center_config.sh" \
/etc/default/domaincheck-ops-center \
http://127.0.0.1:8100 \
http://127.0.0.1:8100 \
"https://${API_DOMAIN}" >/dev/null
cp "${DOMAIN_API_DIR}/deploy/systemd/domain-api.service" /etc/systemd/system/domaincheck-api.service
cp "${DOMAIN_API_DIR}/deploy/systemd/domain-worker.service" /etc/systemd/system/domaincheck-worker.service
@@ -206,3 +215,5 @@ echo "国外主控机部署完成,建议继续执行:"
echo "curl http://127.0.0.1:8100/health"
echo "curl http://127.0.0.1:8100/api/v1/runtime/preflight"
echo "curl http://127.0.0.1:8100/api/v1/runtime/readiness"
echo "bash ${DOMAIN_API_DIR}/deploy/multi-region/drive_ops_center.sh config"
echo "如后续接入大陆 controller请记得修改 /etc/default/domaincheck-ops-center 中的 OPS_MAINLAND_API_BASE_URL"

View File

@@ -0,0 +1,106 @@
#!/usr/bin/env bash
ops_center_normalize_profile_key() {
local raw_value="${1:-default}"
printf '%s' "${raw_value}" | tr '[:lower:]-./' '[:upper:]____' | tr -cd 'A-Z0-9_'
}
ops_center_load_config() {
local caller_dir="${1:-}"
if [[ -z "${caller_dir}" ]]; then
caller_dir="$(pwd)"
fi
local env_active_profile="${OPS_ACTIVE_PROFILE:-}"
local env_profile_names="${OPS_PROFILE_NAMES:-}"
local env_report_root="${OPS_REPORT_ROOT:-}"
local env_mainland_api_base_url="${OPS_MAINLAND_API_BASE_URL:-}"
local env_overseas_api_base_url="${OPS_OVERSEAS_API_BASE_URL:-}"
local env_control_plane_base_url="${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}"
local env_requested_by="${OPS_DEFAULT_REQUESTED_BY:-}"
local env_issued_by="${OPS_DEFAULT_ISSUED_BY:-}"
local env_expires_in_hours="${OPS_DEFAULT_EXPIRES_IN_HOURS:-}"
local env_root_dir="${OPS_DEFAULT_ROOT_DIR:-}"
local env_node_agent_service_name="${OPS_NODE_AGENT_SERVICE_NAME:-}"
local env_node_agent_env_file="${OPS_NODE_AGENT_ENV_FILE:-}"
OPS_CENTER_SCRIPT_DIR="${caller_dir}"
OPS_CENTER_API_DIR="$(cd "${OPS_CENTER_SCRIPT_DIR}/../.." && pwd)"
OPS_CENTER_PROJECT_ROOT="$(cd "${OPS_CENTER_API_DIR}/.." && pwd)"
OPS_CENTER_RUNTIME_DIR="${OPS_CENTER_API_DIR}/runtime"
OPS_CENTER_REPORT_ROOT_DEFAULT="${OPS_CENTER_RUNTIME_DIR}/ops-center-reports"
OPS_CENTER_CONFIG_FILE="${DOMAINCHECK_OPS_CENTER_ENV:-${OPS_CENTER_ENV_FILE:-/etc/default/domaincheck-ops-center}}"
if [[ -f "${OPS_CENTER_CONFIG_FILE}" ]]; then
# shellcheck disable=SC1090
source "${OPS_CENTER_CONFIG_FILE}"
fi
OPS_CENTER_ACTIVE_PROFILE="${OPS_CENTER_PROFILE:-${DOMAINCHECK_OPS_CENTER_PROFILE:-${env_active_profile:-${OPS_ACTIVE_PROFILE:-default}}}}"
OPS_CENTER_ACTIVE_PROFILE_KEY="$(ops_center_normalize_profile_key "${OPS_CENTER_ACTIVE_PROFILE}")"
OPS_CENTER_PROFILE_NAMES="${env_profile_names:-${OPS_PROFILE_NAMES:-${OPS_CENTER_ACTIVE_PROFILE}}}"
OPS_CENTER_REPORT_ROOT="${env_report_root:-${OPS_REPORT_ROOT:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}}"
OPS_CENTER_ENV_MAINLAND_API_BASE_URL="${env_mainland_api_base_url}"
OPS_CENTER_ENV_OVERSEAS_API_BASE_URL="${env_overseas_api_base_url}"
OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL="${env_control_plane_base_url}"
OPS_CENTER_ENV_REQUESTED_BY="${env_requested_by}"
OPS_CENTER_ENV_ISSUED_BY="${env_issued_by}"
OPS_CENTER_ENV_EXPIRES_IN_HOURS="${env_expires_in_hours}"
OPS_CENTER_ENV_ROOT_DIR="${env_root_dir}"
OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME="${env_node_agent_service_name}"
OPS_CENTER_ENV_NODE_AGENT_ENV_FILE="${env_node_agent_env_file}"
}
ops_center_resolve_profile_value() {
local suffix="$1"
local fallback_value="${2:-}"
local profile_key="${OPS_CENTER_ACTIVE_PROFILE_KEY:-DEFAULT}"
local variable_name="OPS_PROFILE_${profile_key}_${suffix}"
local profile_value="${!variable_name:-}"
if [[ -n "${profile_value}" ]]; then
printf '%s' "${profile_value}"
else
printf '%s' "${fallback_value}"
fi
}
ops_center_resolve_mainland_api_base_url() {
ops_center_resolve_profile_value "MAINLAND_API_BASE_URL" "${OPS_CENTER_ENV_MAINLAND_API_BASE_URL:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}"
}
ops_center_resolve_overseas_api_base_url() {
ops_center_resolve_profile_value "OVERSEAS_API_BASE_URL" "${OPS_CENTER_ENV_OVERSEAS_API_BASE_URL:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}"
}
ops_center_resolve_control_plane_base_url() {
ops_center_resolve_profile_value "DEFAULT_CONTROL_PLANE_BASE_URL" "${OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}"
}
ops_center_resolve_requested_by() {
ops_center_resolve_profile_value "DEFAULT_REQUESTED_BY" "${OPS_CENTER_ENV_REQUESTED_BY:-${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}}"
}
ops_center_resolve_issued_by() {
ops_center_resolve_profile_value "DEFAULT_ISSUED_BY" "${OPS_CENTER_ENV_ISSUED_BY:-${OPS_DEFAULT_ISSUED_BY:-ops-center}}"
}
ops_center_resolve_expires_in_hours() {
ops_center_resolve_profile_value "DEFAULT_EXPIRES_IN_HOURS" "${OPS_CENTER_ENV_EXPIRES_IN_HOURS:-${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}}"
}
ops_center_resolve_root_dir() {
ops_center_resolve_profile_value "DEFAULT_ROOT_DIR" "${OPS_CENTER_ENV_ROOT_DIR:-${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}}"
}
ops_center_resolve_node_agent_service_name() {
ops_center_resolve_profile_value "NODE_AGENT_SERVICE_NAME" "${OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME:-${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}}"
}
ops_center_resolve_node_agent_env_file() {
ops_center_resolve_profile_value "NODE_AGENT_ENV_FILE" "${OPS_CENTER_ENV_NODE_AGENT_ENV_FILE:-${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}}"
}
ops_center_ensure_report_dir() {
local report_dir="${1:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}"
mkdir -p "${report_dir}"
}

View File

@@ -0,0 +1,317 @@
#!/usr/bin/env bash
set -euo pipefail
TARGET_PATH="${1:-}"
if [[ -z "${TARGET_PATH}" ]]; then
echo "usage: bash domain-api/deploy/multi-region/review_go_live_bundle.sh <manifest.json|report_dir>" >&2
exit 1
fi
if [[ -d "${TARGET_PATH}" ]]; then
MANIFEST_PATH="${TARGET_PATH%/}/manifest.json"
else
MANIFEST_PATH="${TARGET_PATH}"
fi
if [[ ! -f "${MANIFEST_PATH}" ]]; then
echo "manifest not found: ${MANIFEST_PATH}" >&2
exit 1
fi
python3 - "${MANIFEST_PATH}" <<'PY'
import json
import pathlib
import sys
manifest_path = pathlib.Path(sys.argv[1])
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
artifacts = payload.get("artifacts") or []
failures = payload.get("failures") or []
summary = payload.get("summary") or {}
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
artifact_by_key = {str(item.get("key") or ""): item for item in artifacts if isinstance(item, dict)}
critical_failures = [key for key in critical_keys if not bool((artifact_by_key.get(key) or {}).get("ok", False))]
noncritical_failures = [key for key in failures if key not in critical_keys]
env_audit_artifact = artifact_by_key.get("env_audit") or {}
env_audit_status = ""
env_audit_headline = ""
env_audit_missing_items = []
env_audit_recommended_actions = []
env_audit_blocking = False
env_audit_attention = False
env_audit_runtime_may_need_restart = False
go_live_summary_payload = {}
stack_diagnosis_payload = {}
driver_feed_payload = {}
codex_brief_payload = {}
def load_json_artifact(key: str) -> dict:
artifact = artifact_by_key.get(key) or {}
artifact_path = pathlib.Path(str(artifact.get("path") or ""))
if not artifact_path.is_file():
return {}
try:
loaded = json.loads(artifact_path.read_text(encoding="utf-8"))
except Exception:
return {}
return loaded if isinstance(loaded, dict) else {}
env_audit_path = pathlib.Path(str(env_audit_artifact.get("path") or ""))
if env_audit_path.is_file():
raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace")
marker = "[8/8] condensed env audit summary"
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
if candidate_text:
try:
env_payload = json.loads(candidate_text)
except Exception:
env_payload = {}
env_audit_status = str(env_payload.get("status") or "")
env_audit_headline = str(env_payload.get("headline") or "")
env_audit_missing_items = list(env_payload.get("missing_items") or [])
env_audit_recommended_actions = [str(item).strip() for item in list(env_payload.get("recommended_actions") or []) if str(item).strip()]
env_audit_runtime_may_need_restart = bool(((env_payload.get("runtime") or {}).get("runtime_may_need_restart", False)))
env_audit_blocking = env_audit_status == "blocked"
env_audit_attention = env_audit_status == "attention"
go_live_summary_payload = load_json_artifact("go_live_summary")
stack_diagnosis_payload = load_json_artifact("stack_diagnosis")
driver_feed_payload = load_json_artifact("driver_feed")
codex_brief_payload = load_json_artifact("codex_brief")
go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip()
go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip()
go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip()
go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
stack_diagnosis = stack_diagnosis_payload.get("diagnosis") or {}
stack_launchpad_target_node_code = str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip()
stack_launchpad_recovery_label = str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip()
stack_launchpad_recovery_summary = str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip()
stack_launchpad_bootstrap_pending_nodes = int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
stack_launchpad_acceptance_ready_nodes = int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
driver_feed_summary = driver_feed_payload.get("summary") or {}
codex_brief_summary = codex_brief_payload.get("summary") or {}
def nonempty_distinct(values: dict[str, str]) -> list[str]:
return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()})
def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]:
return sorted({int(values[source]) for source in values if source in available_sources})
launchpad_target_node_codes = {
"go_live_summary": go_live_launchpad_target_node_code,
"stack_diagnosis": stack_launchpad_target_node_code,
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
}
launchpad_recovery_labels = {
"go_live_summary": go_live_launchpad_recovery_label,
"stack_diagnosis": stack_launchpad_recovery_label,
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
}
launchpad_recovery_summaries = {
"go_live_summary": go_live_launchpad_recovery_summary,
"stack_diagnosis": stack_launchpad_recovery_summary,
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
}
launchpad_bootstrap_pending_nodes = {
"go_live_summary": go_live_launchpad_bootstrap_pending_nodes,
"stack_diagnosis": stack_launchpad_bootstrap_pending_nodes,
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
}
launchpad_acceptance_ready_nodes = {
"go_live_summary": go_live_launchpad_acceptance_ready_nodes,
"stack_diagnosis": stack_launchpad_acceptance_ready_nodes,
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
}
available_alignment_sources = {
source
for source, payload_value in {
"go_live_summary": go_live_summary_payload,
"stack_diagnosis": stack_diagnosis_payload,
"driver_feed": driver_feed_payload,
"codex_brief": codex_brief_payload,
}.items()
if isinstance(payload_value, dict) and payload_value
}
launchpad_target_consistent = len(nonempty_distinct({
source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources
})) <= 1
launchpad_recovery_label_consistent = len(nonempty_distinct({
source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources
})) <= 1
launchpad_recovery_summary_consistent = len(nonempty_distinct({
source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources
})) <= 1
launchpad_bootstrap_pending_consistent = len(distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources)) <= 1
launchpad_acceptance_ready_consistent = len(distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources)) <= 1
launchpad_alignment_consistent = all([
launchpad_target_consistent,
launchpad_recovery_label_consistent,
launchpad_recovery_summary_consistent,
launchpad_bootstrap_pending_consistent,
launchpad_acceptance_ready_consistent,
])
node_agent_env_missing = any(
str(item).strip() == "missing_env:/etc/default/domaincheck-node-agent"
for item in env_audit_missing_items
)
if critical_failures or env_audit_blocking:
status = "blocked"
if critical_failures and env_audit_blocking:
headline = "关键报告存在缺失,且环境审计已给出阻断结论,当前不适合上线。"
elif critical_failures:
headline = "上线证据包仍缺关键报告,暂不建议直接宣称收口完成。"
else:
headline = "环境审计已给出阻断结论,需先修复现场环境后再进入发布门禁。"
elif noncritical_failures or env_audit_attention or not launchpad_alignment_consistent:
status = "attention"
if not launchpad_alignment_consistent:
headline = "核心报告已齐,但 launchpad 摘要在不同 surface 之间存在漂移,建议先复核再发版。"
elif noncritical_failures and env_audit_attention:
headline = "核心报告已齐,但补充报告和环境审计都提示仍有缺口,建议先复核再发版。"
elif noncritical_failures:
headline = "核心报告已齐,但仍有补充报告超时或失败,建议先复核再发版。"
else:
headline = "核心报告已齐,但环境审计仍提示基础缺口,建议补齐后再发版。"
else:
status = "ready"
headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
result = {
"report_dir": str(payload.get("report_dir") or manifest_path.parent),
"status": status,
"headline": headline,
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
"critical_failures": critical_failures,
"noncritical_failures": noncritical_failures,
"env_audit": {
"ok": bool(env_audit_artifact.get("ok", False)),
"status": env_audit_status,
"headline": env_audit_headline,
"missing_items": env_audit_missing_items,
"runtime_may_need_restart": env_audit_runtime_may_need_restart,
"recommended_actions": env_audit_recommended_actions,
},
"go_live_summary": {
"go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(),
"publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(),
"operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(),
"next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(),
"launchpad_recommended_target_node_code": go_live_launchpad_target_node_code,
"launchpad_recommended_recovery_label": go_live_launchpad_recovery_label,
"launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes,
},
"stack_diagnosis": {
"stack_status": str((stack_diagnosis_payload.get("diagnosis") or {}).get("stack_status") or "").strip(),
"operator_title": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("operator_decision") or {}).get("title") or "").strip(),
"next_step_action_code": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("next_step") or {}).get("action_code") or "").strip(),
"launchpad_recommended_target_node_code": stack_launchpad_target_node_code,
"launchpad_recommended_recovery_label": stack_launchpad_recovery_label,
"launchpad_recommended_recovery_summary": stack_launchpad_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": stack_launchpad_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": stack_launchpad_acceptance_ready_nodes,
},
"driver_feed": {
"headline": str(driver_feed_payload.get("headline") or "").strip(),
"launch_status": str(((driver_feed_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(),
"launch_ready": bool(((driver_feed_payload.get("automation_coverage") or {}).get("launch_ready", False))),
"launchpad_recommended_target_node_code": str(
((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "")
).strip(),
"launchpad_recommended_recovery_label": str(
((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "")
).strip(),
},
"codex_brief": {
"headline": str(codex_brief_payload.get("headline") or "").strip(),
"focus_entry_key": str(((codex_brief_payload.get("focus") or {}).get("entry_key") or "")).strip(),
"launch_status": str(((codex_brief_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(),
"launch_ready": bool(((codex_brief_payload.get("automation_coverage") or {}).get("launch_ready", False))),
"launchpad_recommended_target_node_code": str(
((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "")
).strip(),
"launchpad_recommended_recovery_label": str(
((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "")
).strip(),
},
"launchpad_alignment": {
"consistent": launchpad_alignment_consistent,
"target_node_code_consistent": launchpad_target_consistent,
"recovery_label_consistent": launchpad_recovery_label_consistent,
"recovery_summary_consistent": launchpad_recovery_summary_consistent,
"bootstrap_pending_consistent": launchpad_bootstrap_pending_consistent,
"acceptance_ready_consistent": launchpad_acceptance_ready_consistent,
"available_sources": sorted(available_alignment_sources),
"target_node_codes": launchpad_target_node_codes,
"recovery_labels": launchpad_recovery_labels,
"recovery_summaries": launchpad_recovery_summaries,
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
},
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
"recommended_next_steps": [
"先看 00_env_audit.txt确认当前机器环境是否存在阻断或注意项",
"先看 02_go_live_summary.json 和 03_stack_diagnosis.json",
"再看 04_driver_feed.json 和 05_codex_brief.json",
"若涉及发布门禁,再看 06_release_launchpad.json",
"若 manifest 里仍有失败项,再看对应 artifact 原文件",
]
+ (
[
"当前缺口指向 Node Agent 未接管:先执行 agent-gap-export拿到首个缺口节点的接管包",
"随后执行 node-bootstrap-plan为缺口节点生成可直接落地的接管脚本",
"如果仓库代码已更新但 bootstrap-plan 输出仍偏旧,先重启控制面 API 再重新导出接管计划",
]
if node_agent_env_missing
else []
)
+ (
[
"env-audit 已识别运行中 API 可能仍是旧代码:先执行 drive_ops_center.sh runtime-refresh-recover按固定链路完成 API 重启与复检",
"若仍需人工逐条确认,再执行 recommended_actions 里的 API 重启与复检命令"
]
if env_audit_runtime_may_need_restart
else []
)
+ (
[
f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code},建议优先执行:{go_live_launchpad_recovery_label or '节点恢复'}",
"先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段,确认三处摘要是否一致",
(go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空,再回到 06_release_launchpad.json 查看 gap rows 详情"),
]
if go_live_launchpad_target_node_code
else []
)
+ (
[
"当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致,先不要直接交付。",
"优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。",
"若只有局部摘要偏旧,先执行 drive_ops_center.sh runtime-refresh-recover并重新导出 go-live bundle。",
]
if not launchpad_alignment_consistent
else []
),
}
print(json.dumps(result, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,15 @@
OPS_CONTROL_PLANE_BASE_URL=http://127.0.0.1:8100
OPS_AGENT_TOKEN=
OPS_AGENT_POLL_INTERVAL_SECONDS=5
NODE_CODE=mainland-worker-01
NODE_REGION=mainland
NODE_ROLE=worker
WORKER_SERVICE_NAME=domaincheck-worker
API_SERVICE_NAME=domaincheck-api
SYNC_AGENT_SERVICE_NAME=domaincheck-sync-agent
NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]
OPS_AGENT_LABELS={}

View File

@@ -0,0 +1,49 @@
# Overseas control-plane unified entrypoint config
#
# Recommended target path:
# /etc/default/domaincheck-ops-center
#
# Usage:
# export DOMAINCHECK_OPS_CENTER_ENV=/etc/default/domaincheck-ops-center
# bash domain-api/deploy/multi-region/drive_ops_center.sh doctor
# Active profile name
OPS_ACTIVE_PROFILE=default
# Optional profile list shown by `drive_ops_center.sh config`
# OPS_PROFILE_NAMES=default,prod,staging
# Mainland controller API base URL
OPS_MAINLAND_API_BASE_URL=http://127.0.0.1:8100
# Overseas control-plane API base URL
OPS_OVERSEAS_API_BASE_URL=http://127.0.0.1:8100
# Public control-plane base URL used in generated node-agent bootstrap plans
OPS_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com
# Default request identity used by CLI wrappers
OPS_DEFAULT_REQUESTED_BY=cli/ops-center
# Default metadata used when generating node-agent bootstrap plans
OPS_DEFAULT_ISSUED_BY=ops-center
OPS_DEFAULT_EXPIRES_IN_HOURS=72
OPS_DEFAULT_ROOT_DIR=/opt/domaincheck
# Optional report export root
# OPS_REPORT_ROOT=/opt/domaincheck/domain-api/runtime/ops-center-reports
# Local node-agent status check defaults
OPS_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
OPS_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent
# Optional per-profile overrides:
# OPS_PROFILE_PROD_MAINLAND_API_BASE_URL=http://10.0.0.10:8100
# OPS_PROFILE_PROD_OVERSEAS_API_BASE_URL=https://api.example.com
# OPS_PROFILE_PROD_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com
# OPS_PROFILE_PROD_DEFAULT_REQUESTED_BY=cli/ops-center-prod
# OPS_PROFILE_PROD_DEFAULT_ISSUED_BY=ops-center-prod
# OPS_PROFILE_PROD_DEFAULT_EXPIRES_IN_HOURS=72
# OPS_PROFILE_PROD_DEFAULT_ROOT_DIR=/opt/domaincheck
# OPS_PROFILE_PROD_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
# OPS_PROFILE_PROD_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent

View File

@@ -0,0 +1,17 @@
[Unit]
Description=domainCheck Node Agent
After=network.target
[Service]
Type=simple
WorkingDirectory=/opt/domaincheck/domain-api
EnvironmentFile=-/etc/default/domaincheck-node-agent
Environment="PATH=/home/www/.nvm/versions/node/v20.20.2/bin:/home/www/.local/bin:/home/www/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m app.node_agent
Restart=always
RestartSec=5
User=www
Group=www
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,98 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.core.config import settings
from app.services.build_info_service import (
_expected_route_paths,
get_runtime_build_info,
remember_registered_route_paths,
)
class BuildInfoServiceTests(unittest.TestCase):
def tearDown(self) -> None:
remember_registered_route_paths([])
def test_runtime_build_info_prefers_configured_manifest_and_reports_registered_surface(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
manifest_path = Path(temp_dir) / "release_manifest.json"
manifest_path.write_text(
json.dumps(
{
"package_name": "domaincheck_release_demo",
"generated_at": "2026-04-18T12:00:00",
"commit_sha": "bd6bcb2",
"commit_ref": "main",
"sha256": "demo-sha256",
},
ensure_ascii=False,
),
encoding="utf-8",
)
route_paths = list(_expected_route_paths().values())
remember_registered_route_paths(route_paths)
with patch.multiple(
settings,
build_manifest_path=str(manifest_path),
build_commit_sha="",
build_commit_ref="",
build_generated_at="",
build_package_name="",
build_checksum="",
build_source_label="",
):
payload = get_runtime_build_info()
self.assertEqual("configured_manifest", payload["source"])
self.assertEqual("domaincheck_release_demo", payload["package_name"])
self.assertEqual("bd6bcb2", payload["commit_sha"])
self.assertEqual("main", payload["commit_ref"])
self.assertEqual("demo-sha256", payload["checksum"])
self.assertEqual(str(manifest_path), payload["manifest_path"])
self.assertTrue(payload["route_surface"]["surface_complete"])
self.assertEqual([], payload["route_surface"]["missing_keys"])
self.assertEqual("registered", payload["route_surface"]["mode"])
self.assertIn("repository_capabilities", payload)
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
self.assertIn("supports_multi_layout_bootstrap", payload["repository_capabilities"])
def test_runtime_build_info_env_override_wins_and_missing_routes_are_visible(self) -> None:
with patch.multiple(
settings,
build_manifest_path="",
build_commit_sha="246838ae4c07",
build_commit_ref="main",
build_generated_at="2026-04-18T12:10:00",
build_package_name="domaincheck_release_env",
build_checksum="env-sha256",
build_source_label="env-override",
):
payload = get_runtime_build_info(route_paths=[_expected_route_paths()["ops_contracts"]])
self.assertEqual("env-override", payload["source"])
self.assertEqual("domaincheck_release_env", payload["package_name"])
self.assertEqual("246838ae4c07", payload["commit_sha"])
self.assertEqual("main", payload["commit_ref"])
self.assertEqual("env-sha256", payload["checksum"])
self.assertFalse(payload["route_surface"]["surface_complete"])
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_acceptance_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_recovery_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_recovery_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_scene_log", payload["route_surface"]["missing_keys"])
self.assertIn("runtime_build_info", payload["route_surface"]["missing_keys"])
self.assertEqual("registered", payload["route_surface"]["mode"])
self.assertIn("repository_capabilities", payload)
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,111 @@
from __future__ import annotations
import os
import re
import subprocess
import unittest
from pathlib import Path
class DriveOpsCenterDocsTests(unittest.TestCase):
def test_authoritative_docs_only_reference_supported_commands(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
doc_paths = [
repo_root / "domain-api" / "deploy" / "multi-region" / "README.md",
repo_root / "docs" / "25_domainCheck_海外单脑控制面上线收口总表.md",
repo_root / "docs" / "26_domainCheck_发布前运行验证与交付模板.md",
]
help_output = subprocess.run(
["bash", str(script_path), "help"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout
supported_commands = self._extract_supported_commands(help_output)
documented_commands = self._extract_documented_commands(doc_paths)
missing_commands = sorted(documented_commands - supported_commands)
self.assertEqual(
[],
missing_commands,
msg=(
"These drive_ops_center.sh commands are referenced by authoritative docs "
"but missing from the CLI help output: "
+ ", ".join(missing_commands)
),
)
self.assertIn("stack-diagnosis", documented_commands)
self.assertIn("stack-diagnosis", supported_commands)
def test_stack_diagnosis_summary_argument_normalization_does_not_treat_summary_as_host(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
env = dict(os.environ)
env["DOMAINCHECK_OPS_CENTER_ENV"] = "/tmp/domaincheck-ops-center-does-not-exist"
env["OPS_MAINLAND_API_BASE_URL"] = "http://127.0.0.1:9"
result = subprocess.run(
["bash", str(script_path), "stack-diagnosis", "summary"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
combined_output = f"{result.stdout}\n{result.stderr}"
self.assertNotIn("Could not resolve host: summary", combined_output)
self.assertNotIn("NameError", combined_output)
self.assertIn("[9/9] condensed stack summary", combined_output)
def test_env_drift_recommends_stack_diagnosis_instead_of_stack_check(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "check_env_drift.sh"
script_text = script_path.read_text(encoding="utf-8")
self.assertIn(
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")',
script_text,
)
self.assertNotIn(
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-check http://127.0.0.1:8100 summary")',
script_text,
)
@staticmethod
def _extract_supported_commands(help_output: str) -> set[str]:
commands: set[str] = set()
in_commands_section = False
for raw_line in help_output.splitlines():
line = raw_line.rstrip()
if line.strip() == "commands:":
in_commands_section = True
continue
if line.strip() == "examples:":
break
if not in_commands_section:
continue
stripped = line.strip()
if not stripped:
continue
command = stripped.split()[0]
if re.fullmatch(r"[a-z0-9-]+", command):
commands.add(command)
return commands
@staticmethod
def _extract_documented_commands(doc_paths: list[Path]) -> set[str]:
pattern = re.compile(r"drive_ops_center\.sh\s+([a-z0-9-]+)")
commands: set[str] = set()
for doc_path in doc_paths:
text = doc_path.read_text(encoding="utf-8")
commands.update(match.group(1) for match in pattern.finditer(text))
return commands
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,284 @@
import json
import os
import tempfile
import unittest
import urllib.error
from unittest.mock import patch
from app import node_agent
class NodeAgentDeliveryQueueTests(unittest.TestCase):
def test_base_payload_includes_delivery_queue_snapshot(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
node_agent._ensure_queue_dirs()
payload = node_agent._base_payload()
delivery_queue = payload["metadata"]["delivery_queue"]
self.assertEqual("healthy", delivery_queue["state"])
self.assertEqual(0, delivery_queue["pending_count"])
self.assertEqual(0, delivery_queue["dead_letter_count"])
def test_job_event_network_failure_is_queued_for_retry(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
node_agent,
"_post",
side_effect=urllib.error.URLError("offline"),
):
result = node_agent._job_event(
7,
event_type="executor_received",
message="accepted",
client_event_id="evt-001",
summary_text="已接单",
focus_ref={"kind": "ops_job", "job_id": 7},
)
self.assertEqual("queued", result["state"])
pending_dir = os.path.join(temp_dir, "pending")
pending_files = sorted(os.listdir(pending_dir))
self.assertEqual(1, len(pending_files))
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
record = json.load(handle)
self.assertEqual("evt-001", record["payload"]["client_event_id"])
self.assertEqual("已接单", record["payload"]["summary_text"])
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
self.assertEqual("job_event", record["kind"])
def test_job_complete_transport_failure_preserves_client_request_id(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
node_agent,
"_post",
side_effect=urllib.error.URLError("offline"),
):
result = node_agent._job_complete(
9,
status="success",
stdout="done",
stderr="",
result={"ok": True},
client_request_id="complete-001",
duration_ms=321,
summary_text="执行成功",
focus_ref={"kind": "ops_job", "job_id": 9},
)
self.assertEqual("queued", result["state"])
pending_dir = os.path.join(temp_dir, "pending")
pending_files = sorted(os.listdir(pending_dir))
self.assertEqual(1, len(pending_files))
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
record = json.load(handle)
self.assertEqual("complete-001", record["payload"]["client_request_id"])
self.assertEqual(321, record["payload"]["duration_ms"])
self.assertEqual("执行成功", record["payload"]["summary_text"])
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
self.assertEqual("job_complete", record["kind"])
def test_flush_delivery_queue_delivers_pending_records(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_event",
"/api/v1/ops/agent/jobs/7/events",
{
"node_code": "mainland-worker-01",
"event_type": "executor_received",
"message": "accepted",
"client_event_id": "evt-002",
},
"evt-002",
)
node_agent._store_pending_delivery(record)
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
summary = node_agent._flush_delivery_queue(limit=10)
self.assertEqual(1, summary["delivered"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
def test_flush_delivery_queue_moves_semantic_failure_to_dead_letter(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_complete",
"/api/v1/ops/agent/jobs/9/complete",
{
"node_code": "mainland-worker-01",
"status": "success",
"client_request_id": "complete-002",
},
"complete-002",
)
node_agent._store_pending_delivery(record)
with patch.object(
node_agent,
"_post",
return_value={
"code": 1,
"message": "任务不存在或不属于当前节点",
"detail_code": "ops_job_not_owned_by_agent",
},
):
summary = node_agent._flush_delivery_queue(limit=10)
self.assertEqual(1, summary["dead_letter"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
self.assertEqual(1, len(os.listdir(os.path.join(temp_dir, "dead-letter"))))
def test_replay_dead_letter_records_moves_record_back_to_pending_and_flushes(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_event",
"/api/v1/ops/agent/jobs/7/events",
{
"node_code": "mainland-worker-01",
"event_type": "executor_received",
"message": "accepted",
"client_event_id": "evt-003",
},
"evt-003",
)
node_agent._store_dead_letter_delivery(
record,
reason="temporary failure",
detail_code="transient_error",
)
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
ok, message, data = node_agent._execute_action(
"delivery.queue.replay",
{
"selector": {"record_id": "evt-003"},
"limit": 10,
"flush_after_replay": True,
"reason": "恢复后重放",
},
)
self.assertTrue(ok)
self.assertEqual("已重放 1 条死信记录", message)
self.assertEqual(1, data["replayed_total"])
self.assertEqual(1, data["flush_summary"]["delivered"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
def test_discard_dead_letter_records_archives_record(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_complete",
"/api/v1/ops/agent/jobs/8/complete",
{
"node_code": "mainland-worker-01",
"status": "failed",
"client_request_id": "complete-003",
},
"complete-003",
)
node_agent._store_dead_letter_delivery(
record,
reason="permanent failure",
detail_code="ops_job_invalid_status",
)
ok, message, data = node_agent._execute_action(
"delivery.queue.discard",
{
"selector": {"record_id": "complete-003"},
"limit": 10,
"discarded_by": "web-ui",
"reason": "确认不需要再补发",
},
)
self.assertTrue(ok)
self.assertEqual("已丢弃 1 条死信记录", message)
self.assertEqual(1, data["discarded_total"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
discarded_files = os.listdir(os.path.join(temp_dir, "discarded"))
self.assertEqual(1, len(discarded_files))
with open(os.path.join(temp_dir, "discarded", discarded_files[0]), "r", encoding="utf-8") as handle:
archived = json.load(handle)
self.assertEqual("web-ui", archived["discarded_by"])
self.assertEqual("确认不需要再补发", archived["discard_reason"])
def test_process_job_passes_envelope_context_to_completion(self) -> None:
job = {
"id": 12,
"job_code": "ops-20260418-000012",
"job_type": "release_deploy",
"action": "logs.collect",
"payload": {"service_name": "domaincheck-worker", "lines": 120},
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"focus_ref": {"kind": "ops_job", "job_id": 12, "job_code": "ops-20260418-000012"},
"release_context": {"release_version": "v2026.04.18", "rollout_id": 3},
"step_ref": {"step_id": 22, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
}
with patch.object(node_agent, "_job_start") as mock_job_start, patch.object(
node_agent,
"_job_event",
) as mock_job_event, patch.object(
node_agent,
"_execute_action",
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
) as mock_execute_action, patch.object(
node_agent,
"_job_complete",
return_value={"state": "sent"},
) as mock_job_complete:
node_agent._process_job(job)
self.assertEqual("worker_logs", mock_job_start.call_args.kwargs["job"]["step_key"])
self.assertEqual("ops_job", mock_job_event.call_args.kwargs["focus_ref"]["kind"])
self.assertEqual("已接单 logs.collect", mock_job_event.call_args.kwargs["summary_text"])
self.assertEqual("logs.collect", mock_execute_action.call_args.args[0])
self.assertEqual("v2026.04.18", mock_job_complete.call_args.kwargs["release_context"]["release_version"])
self.assertEqual("ops_job", mock_job_complete.call_args.kwargs["focus_ref"]["kind"])
self.assertEqual("收集完成", mock_job_complete.call_args.kwargs["summary_text"])
self.assertGreaterEqual(mock_job_complete.call_args.kwargs["duration_ms"], 0)
def test_process_job_continues_when_job_start_delivery_fails(self) -> None:
job = {
"id": 18,
"job_code": "ops-20260418-000018",
"job_type": "ops_action",
"action": "logs.collect",
"payload": {"service_name": "domaincheck-worker", "lines": 50},
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"focus_ref": {"kind": "ops_job", "job_id": 18, "job_code": "ops-20260418-000018"},
"release_context": {},
"step_ref": {"step_id": 28, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
}
with patch.object(
node_agent,
"_job_start",
side_effect=urllib.error.URLError("temporary offline"),
) as mock_job_start, patch.object(
node_agent,
"_job_event",
return_value={"state": "queued"},
) as mock_job_event, patch.object(
node_agent,
"_execute_action",
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
) as mock_execute_action, patch.object(
node_agent,
"_job_complete",
return_value={"state": "sent"},
) as mock_job_complete:
node_agent._process_job(job)
mock_job_start.assert_called_once()
mock_execute_action.assert_called_once()
mock_job_complete.assert_called_once()
event_payload = mock_job_event.call_args.kwargs["payload"]
self.assertEqual("failed_local", event_payload["start_delivery_state"])
self.assertIn("temporary offline", event_payload["start_delivery_error"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,41 @@
import unittest
from app import node_agent
class NodeAgentReleaseNormalizationTests(unittest.TestCase):
def test_normalize_text_list_supports_strings_and_lists(self) -> None:
self.assertEqual(
["domaincheck-api", "domaincheck-worker"],
node_agent._normalize_text_list("domaincheck-api\ndomaincheck-worker"),
)
self.assertEqual(
["domaincheck-api", "domaincheck-worker"],
node_agent._normalize_text_list(["domaincheck-api", " domaincheck-worker "]),
)
self.assertEqual([], node_agent._normalize_text_list(""))
def test_health_check_services_fall_back_to_restart_services_when_missing(self) -> None:
result = node_agent._normalize_release_health_check_services(
{},
["domaincheck-worker"],
)
self.assertEqual(["domaincheck-worker"], result)
def test_health_check_services_can_be_explicitly_empty(self) -> None:
result = node_agent._normalize_release_health_check_services(
{"health_check_services": ""},
["domaincheck-worker"],
)
self.assertEqual([], result)
def test_health_check_services_accept_multiline_string(self) -> None:
result = node_agent._normalize_release_health_check_services(
{"health_check_services": "domaincheck-api\ndomaincheck-sync-agent"},
["domaincheck-worker"],
)
self.assertEqual(["domaincheck-api", "domaincheck-sync-agent"], result)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,115 @@
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_action_executor_core import build_service_name_map, execute_structured_action
class _FakeRunner:
def __init__(self, responses: dict[tuple[str, ...], tuple[int, str, str]]) -> None:
self.responses = responses
self.calls: list[tuple[str, ...]] = []
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
key = tuple(command)
self.calls.append(key)
return self.responses.get(key, (0, "", ""))
class OpsActionExecutorCoreTests(unittest.TestCase):
def test_health_snapshot_includes_host_context_and_ok_flags(self) -> None:
runner = _FakeRunner(
{
("systemctl", "is-active", "domaincheck-api"): (0, "active", ""),
("systemctl", "is-active", "domaincheck-worker"): (3, "inactive", ""),
("systemctl", "is-active", "domaincheck-sync-agent"): (0, "active", ""),
("systemctl", "is-active", "domaincheck-node-agent"): (0, "active", ""),
}
)
ok, message, data = execute_structured_action(
"health.snapshot",
{},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
host_context={"hostname": "mainland-controller-01", "ip": "121.204.244.188"},
)
self.assertTrue(ok)
self.assertEqual("health snapshot collected", message)
self.assertEqual("mainland-controller-01", data["hostname"])
self.assertEqual("121.204.244.188", data["ip"])
self.assertTrue(data["checks"]["api"]["ok"])
self.assertFalse(data["checks"]["worker"]["ok"])
self.assertEqual(4, len(runner.calls))
def test_logs_collect_defaults_to_worker_service_and_clamps_lines(self) -> None:
runner = _FakeRunner(
{
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"): (
0,
"line-1\nline-2",
"",
)
}
)
ok, message, data = execute_structured_action(
"logs.collect",
{"lines": 9999},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
)
self.assertTrue(ok)
self.assertEqual("line-1\nline-2", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual(500, data["lines"])
self.assertEqual(
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"),
runner.calls[0],
)
def test_logs_collect_falls_back_to_worker_log_file_when_journal_is_denied(self) -> None:
runner = _FakeRunner(
{
("journalctl", "-u", "domaincheck-worker", "-n", "120", "--no-pager"): (
1,
"",
"No journal files were opened due to insufficient permissions.",
)
}
)
with tempfile.TemporaryDirectory() as temp_dir:
log_path = Path(temp_dir) / "detect_worker.log"
log_path.write_text("worker-a\nworker-b\n", encoding="utf-8")
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
ok, message, data = execute_structured_action(
"logs.collect",
{"lines": 120},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
)
self.assertTrue(ok)
self.assertEqual("worker-a\nworker-b", message)
self.assertEqual("file_fallback", data["collection_source"])
self.assertTrue(data["fallback_used"])
self.assertIn("detect_worker.log", data["log_paths"][0])
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,96 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.api.routes.ops import (
ops_doctor_decision,
ops_go_live_bundle,
ops_go_live_review,
ops_go_live_signoff,
ops_go_live_summary,
)
class OpsApiRoutesTestCase(unittest.TestCase):
@patch("app.api.routes.ops.get_ops_go_live_signoff")
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
mock_get_ops_go_live_signoff.return_value = {
"signoff_status": "ready",
"headline": "ok",
}
response = ops_go_live_signoff(base_url="http://127.0.0.1:8100", report_dir="/tmp/signoff")
self.assertEqual(0, response.code)
self.assertEqual("ready", response.data["signoff_status"])
self.assertEqual("ok", response.data["headline"])
mock_get_ops_go_live_signoff.assert_called_once_with(
base_url="http://127.0.0.1:8100",
report_dir="/tmp/signoff",
)
@patch("app.api.routes.ops.get_ops_go_live_summary")
def test_go_live_summary_route_passes_base_url(self, mock_get_ops_go_live_summary) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
}
response = ops_go_live_summary(base_url="http://example.test")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["go_live_status"])
mock_get_ops_go_live_summary.assert_called_once_with(base_url="http://example.test")
@patch("app.api.routes.ops.get_ops_go_live_bundle")
def test_go_live_bundle_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_bundle) -> None:
mock_get_ops_go_live_bundle.return_value = {
"status": "missing",
}
response = ops_go_live_bundle(base_url="http://example.test", report_dir="/tmp/go-live")
self.assertEqual(0, response.code)
self.assertEqual("missing", response.data["status"])
mock_get_ops_go_live_bundle.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/go-live",
)
@patch("app.api.routes.ops.get_ops_go_live_review")
def test_go_live_review_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_review) -> None:
mock_get_ops_go_live_review.return_value = {
"status": "attention",
"headline": "review",
}
response = ops_go_live_review(base_url="http://example.test", report_dir="/tmp/review")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["status"])
self.assertEqual("review", response.data["headline"])
mock_get_ops_go_live_review.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/review",
)
@patch("app.api.routes.ops.get_ops_doctor_decision")
def test_doctor_decision_route_passes_base_url_and_report_dir(self, mock_get_ops_doctor_decision) -> None:
mock_get_ops_doctor_decision.return_value = {
"status": "attention",
"headline": "doctor",
}
response = ops_doctor_decision(base_url="http://example.test", report_dir="/tmp/doctor")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["status"])
self.assertEqual("doctor", response.data["headline"])
mock_get_ops_doctor_decision.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/doctor",
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,44 @@
import unittest
from app.services.ops_command_service import build_bash_command, ops_script_path
class OpsCommandServiceTests(unittest.TestCase):
def test_ops_script_path_prefixes_multi_region_directory(self) -> None:
self.assertEqual(
"domain-api/deploy/multi-region/drive_ops_center.sh",
ops_script_path("drive_ops_center.sh"),
)
def test_ops_script_path_normalizes_leading_slash(self) -> None:
self.assertEqual(
"domain-api/deploy/multi-region/check_ops_center_stack.sh",
ops_script_path("/check_ops_center_stack.sh"),
)
def test_build_bash_command_joins_non_empty_args(self) -> None:
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
build_bash_command(
"drive_ops_center.sh",
"driver-resolve",
"http://127.0.0.1:8100",
"handover_first_gap",
),
)
def test_build_bash_command_skips_blank_args(self) -> None:
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad",
build_bash_command(
"drive_release_hub.sh",
"http://127.0.0.1:8100",
"",
"launchpad",
" ",
),
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,391 @@
import json
import unittest
from datetime import datetime
from unittest.mock import MagicMock, patch
from app.core.config import settings
from app.services.ops_job_service import (
_execute_control_plane_job,
_serialize_job_row,
create_ops_job,
dispatch_ops_job,
upsert_managed_node,
)
class OpsJobServiceTests(unittest.TestCase):
def test_serialize_job_row_adds_execution_mode_label(self) -> None:
row = (
9,
"ops-2026041801-abcd12",
"logs.collect",
"node",
"mainland-worker-01",
"queued",
"remote-agent",
"api",
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 0, 0),
None,
None,
datetime(2026, 4, 18, 10, 0, 0),
"low",
False,
"approved",
"",
None,
"",
"",
None,
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
job = _serialize_job_row(row)
self.assertEqual("remote-agent", job["execution_mode"])
self.assertEqual("远端 Agent", job["execution_mode_label"])
self.assertEqual("排队中", job["status_label"])
self.assertEqual("已审批", job["approval_status_label"])
self.assertEqual(["mainland-worker-01"], job["target_node_codes"])
self.assertEqual(0, job["steps_total"])
self.assertFalse(job["steps_loaded"])
self.assertFalse(job["is_compact"])
self.assertEqual("ops_job", job["focus_ref"]["kind"])
self.assertEqual(9, job["focus_ref"]["job_id"])
self.assertIn("目标节点 mainland-worker-01", job["summary_text"])
def test_serialize_job_row_counts_loaded_steps(self) -> None:
row = (
10,
"ops-2026041801-efgh34",
"diagnostics.collect",
"node",
"mainland-worker-02",
"running",
"ssh",
"web-ui",
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 5, 0),
datetime(2026, 4, 18, 10, 5, 2),
None,
datetime(2026, 4, 18, 10, 5, 3),
"medium",
True,
"approved",
"admin",
datetime(2026, 4, 18, 10, 5, 1),
"",
"",
datetime(2026, 4, 18, 10, 5, 2),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
job = _serialize_job_row(
row,
steps=[
{"id": 1, "status": "success"},
{"id": 2, "status": "running"},
{"id": 3, "status": "failed"},
],
)
self.assertEqual("执行中", job["status_label"])
self.assertEqual("已审批", job["approval_status_label"])
self.assertEqual(3, job["steps_total"])
self.assertEqual(1, job["steps_running"])
self.assertEqual(1, job["steps_success"])
self.assertEqual(1, job["steps_failed"])
self.assertEqual(2, job["steps_terminal"])
self.assertTrue(job["steps_loaded"])
self.assertIn("mainland-worker-02", job["summary_text"])
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan")
def test_execute_control_plane_bootstrap_uses_managed_node_defaults(
self,
mock_build_node_agent_bootstrap_plan,
mock_list_managed_nodes,
) -> None:
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
}
]
mock_build_node_agent_bootstrap_plan.return_value = (
True,
"节点 Agent 接入方案已生成",
{
"token_preview": "agent-...1234",
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
},
},
)
ok, message, data = _execute_control_plane_job(
"node.bootstrap",
target_node_code="mainland-worker-02",
payload={
"root_dir": "/opt/domaincheck",
"expires_in_hours": 12,
},
requested_by="unit-test",
metadata={"source": "test"},
)
self.assertTrue(ok)
self.assertEqual("节点 Agent 接入方案已生成", message)
self.assertEqual("control-plane", data["execution_mode"])
self.assertEqual("mainland", data["node_region"])
self.assertEqual("worker", data["node_role"])
mock_build_node_agent_bootstrap_plan.assert_called_once_with(
node_code="mainland-worker-02",
node_region="mainland",
node_role="worker",
issued_by="unit-test",
expires_in_hours=12,
control_plane_base_url="",
root_dir="/opt/domaincheck",
metadata={
"source": "test",
"issued_from": "ops-job/control-plane",
"target_node_code": "mainland-worker-02",
"node_region": "mainland",
"node_role": "worker",
},
)
@patch("app.services.ops_job_service.get_db")
def test_upsert_managed_node_preserves_remote_fields_when_sync_payload_omits_them(self, mock_get_db) -> None:
existing_metadata = {"operator_note": "keep-me"}
merged_metadata = {"operator_note": "keep-me", "cluster_status": "online"}
existing_row = (
"mainland-worker-01",
"mainland",
"worker",
"Mainland Worker 01",
"121.204.244.248",
22,
"root",
"key",
"stable",
False,
json.dumps(existing_metadata, ensure_ascii=False),
datetime(2026, 4, 18, 1, 0, 0),
None,
datetime(2026, 4, 18, 1, 5, 0),
)
returned_row = (
"mainland-worker-01",
"mainland",
"worker",
"S244-248",
"121.204.244.248",
22,
"root",
"key",
"stable",
False,
json.dumps(merged_metadata, ensure_ascii=False),
datetime(2026, 4, 18, 1, 0, 0),
None,
datetime(2026, 4, 18, 1, 6, 0),
)
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.side_effect = [existing_row, returned_row]
ok, message, data = upsert_managed_node(
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"title": "S244-248",
"metadata": {
"cluster_status": "online",
},
}
)
self.assertTrue(ok)
self.assertEqual("托管节点已保存", message)
node = data["node"]
self.assertEqual("121.204.244.248", node["ssh_host"])
self.assertEqual("root", node["ssh_user"])
self.assertFalse(node["is_enabled"])
self.assertEqual("keep-me", node["metadata"]["operator_note"])
self.assertEqual("online", node["metadata"]["cluster_status"])
insert_call = cursor.execute.call_args_list[-1]
insert_params = insert_call.args[1]
self.assertEqual("121.204.244.248", insert_params[4])
self.assertEqual(22, insert_params[5])
self.assertEqual("root", insert_params[6])
self.assertFalse(insert_params[9])
self.assertEqual(merged_metadata, json.loads(insert_params[10]))
@patch("app.services.ops_agent_service.append_ops_job_event")
@patch("app.services.ops_job_service._execute_ssh_job_record")
@patch("app.services.ops_job_service.get_db")
def test_dispatch_ops_job_ssh_executes_via_ssh_record(
self,
mock_get_db,
mock_execute_ssh_job_record,
mock_append_ops_job_event,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.return_value = (
9,
"ops-2026041801-abcd12",
"logs.collect",
"node",
"mainland-worker-01",
"queued",
"ssh",
"api",
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 0, 0),
None,
None,
datetime(2026, 4, 18, 10, 0, 0),
"low",
False,
"approved",
"",
None,
"",
"",
None,
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
mock_execute_ssh_job_record.return_value = (
True,
"worker logs collected",
{"job": {"id": 9, "status": "success"}},
)
ok, message, data = dispatch_ops_job(9)
self.assertTrue(ok)
self.assertEqual("已按 SSH 执行派发worker logs collected", message)
self.assertEqual("success", data["job"]["status"])
mock_execute_ssh_job_record.assert_called_once_with(9)
mock_append_ops_job_event.assert_called_once()
event_kwargs = mock_append_ops_job_event.call_args.kwargs
self.assertEqual("job_dispatch_requested", event_kwargs["event_type"])
self.assertEqual("mainland-worker-01", event_kwargs["node_code"])
self.assertEqual("ssh", event_kwargs["payload"]["execution_mode"])
@patch("app.services.ops_agent_service.append_ops_job_event")
@patch("app.services.ops_job_service.get_ops_job")
@patch("app.services.ops_job_service._execute_local_job")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_job_service.get_db")
def test_create_ops_job_auto_uses_local_runtime_for_local_structured_action(
self,
mock_get_db,
mock_preview_ops_job_policy,
mock_execute_local_job,
mock_get_ops_job,
mock_append_ops_job_event,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.return_value = (42,)
mock_preview_ops_job_policy.return_value = {
"risk_level": "low",
"approval_required": False,
"blocked": False,
"blocking_reasons": [],
}
mock_execute_local_job.return_value = (
True,
"worker logs collected",
{"executor": "local-runtime"},
)
mock_get_ops_job.return_value = {
"id": 42,
"status": "success",
"execution_mode": "local-runtime",
}
ok, message, data = create_ops_job(
{
"action": "logs.collect",
"target_node_code": settings.node_code,
"payload": {
"service_name": "domaincheck-worker",
"lines": 120,
},
"requested_by": "unit-test",
}
)
self.assertTrue(ok)
self.assertEqual("worker logs collected", message)
self.assertTrue(data["executed_immediately"])
self.assertEqual("local-runtime", data["job"]["execution_mode"])
mock_execute_local_job.assert_called_once_with(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 120,
},
)
insert_call = next(
call
for call in cursor.execute.call_args_list
if "INSERT INTO ops_jobs" in str(call.args[0])
)
insert_params = insert_call.args[1]
self.assertEqual("local-runtime", insert_params[5])
mock_append_ops_job_event.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,220 @@
import unittest
from unittest.mock import patch
from app.services.ops_playbook_service import (
get_recent_ops_playbook_runs,
list_ops_playbook_run_events,
preview_ops_playbook,
)
class OpsPlaybookServiceTests(unittest.TestCase):
def test_onboarding_bootstrap_preview_uses_control_plane(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "onboarding.bootstrap",
"node_codes": ["mainland-worker-02"],
}
)
self.assertTrue(ok, message)
self.assertEqual("playbook 预览生成成功", message)
self.assertEqual("onboarding.bootstrap", data["playbook"]["key"])
self.assertEqual("control-plane", data["execution_mode"])
self.assertEqual("控制面", data["execution_mode_label"])
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
self.assertEqual("控制面", data["playbook"]["default_execution_mode_label"])
self.assertEqual("控制面", data["playbook"]["execution_mode_options"][0]["label"])
self.assertTrue(data["auto_approve"])
self.assertEqual(1, data["step_count"])
step = data["steps"][0]
self.assertEqual("bootstrap", step["step_key"])
self.assertEqual("node.bootstrap", step["template_key"])
self.assertEqual("node.bootstrap", step["action"])
self.assertEqual("control-plane", step["execution_mode"])
self.assertEqual(1, step["expected_jobs"])
def test_scene_logs_key_preview_allows_ssh_execution_mode(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "scene.logs.key",
"node_codes": ["mainland-worker-01"],
"execution_mode": "ssh",
}
)
self.assertTrue(ok, message)
self.assertEqual("playbook 预览生成成功", message)
self.assertEqual("ssh", data["execution_mode"])
self.assertEqual("SSH", data["execution_mode_label"])
self.assertEqual(["remote-agent", "ssh"], data["playbook"]["execution_modes"])
self.assertEqual("ssh", data["steps"][0]["execution_mode"])
self.assertEqual("SSH", data["steps"][0]["execution_mode_label"])
def test_onboarding_bootstrap_preview_rejects_unsupported_execution_mode(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "onboarding.bootstrap",
"node_codes": ["mainland-worker-02"],
"execution_mode": "ssh",
}
)
self.assertFalse(ok)
self.assertEqual("该 playbook 仅支持 control-plane", message)
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_playbook_service.list_ops_jobs")
def test_recent_onboarding_bootstrap_run_focus_summary_reflects_agent_state(
self,
mock_list_ops_jobs,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_ops_jobs.return_value = [
{
"id": 31,
"job_code": "ops-bootstrap-31",
"action": "node.bootstrap",
"status": "success",
"target_node_code": "mainland-worker-02",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
"metadata": {
"playbook": {
"run_code": "pbr-bootstrap-1",
"key": "onboarding.bootstrap",
"title": "生成节点接入工单",
"group_key": "onboarding",
"group_title": "接管验收",
"requested_by": "tester",
"execution_mode": "control-plane",
"step_key": "bootstrap",
"step_title": "生成 Node Agent 接入工单",
"step_order": 1,
"step_count": 1,
"expected_jobs_total": 1,
"target_nodes_total": 1,
"template_key": "node.bootstrap",
}
},
"result": {
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
}
},
}
]
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"agent_state": "pending_bootstrap",
"is_managed": True,
}
]
}
payload = get_recent_ops_playbook_runs(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["runs"]))
run = payload["runs"][0]
self.assertEqual("success", run["status"])
self.assertEqual("成功", run["status_label"])
self.assertEqual("控制面", run["execution_mode_label"])
self.assertEqual("warning", run["focus_level"])
self.assertIn("待执行 bootstrap 脚本", run["focus_summary"])
self.assertEqual(run["focus_summary"], run["summary_text"])
self.assertEqual(1, run["steps_total"])
self.assertEqual("pbr-bootstrap-1", run["focus_ref"]["run_code"])
self.assertEqual("生成 Node Agent 接入工单", run["steps"][0]["title"])
self.assertEqual("成功", run["steps"][0]["status_label"])
self.assertTrue(run["steps"][0]["summary_text"])
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_playbook_service.list_ops_job_events_for_jobs")
@patch("app.services.ops_playbook_service.list_ops_jobs")
def test_playbook_run_events_add_contract_aliases(
self,
mock_list_ops_jobs,
mock_list_ops_job_events_for_jobs,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_list_ops_jobs.return_value = [
{
"id": 41,
"job_code": "ops-inspection-41",
"action": "logs.collect",
"status": "running",
"target_node_code": "mainland-worker-01",
"created_at": "2026-04-18 06:30:00",
"updated_at": "2026-04-18 06:31:00",
"metadata": {
"playbook": {
"run_code": "pbr-inspection-1",
"key": "inspection.standard",
"title": "标准巡检",
"group_key": "diagnostics",
"group_title": "标准巡检",
"requested_by": "tester",
"execution_mode": "remote-agent",
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"step_order": 2,
"step_count": 3,
"expected_jobs_total": 3,
"target_nodes_total": 1,
"template_key": "logs.collect",
}
},
}
]
mock_list_ops_job_events_for_jobs.return_value = [
{
"id": 501,
"event_key": "job-event:501",
"job_id": 41,
"node_code": "mainland-worker-01",
"event_type": "job_running",
"level": "info",
"level_label": "信息",
"message": "开始收集 Worker 日志",
"summary": "开始收集 Worker 日志",
"summary_text": "远端 Agent 已开始收集 Worker 日志",
"focus_ref": {
"kind": "ops_job",
"job_id": 41,
"job_code": "ops-inspection-41",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
"event_key": "job-event:501",
"step_key": "worker_logs",
},
"payload": {},
"created_at": "2026-04-18 06:31:00",
"occurred_at": "2026-04-18 06:31:00",
"ui_intent": {"kind": "job_detail", "job_id": 41, "focus_event_id": 501},
}
]
payload = list_ops_playbook_run_events("pbr-inspection-1", limit=10)
self.assertEqual("inspection.standard", payload["playbook_run"]["playbook_key"])
self.assertEqual(1, len(payload["events"]))
event = payload["events"][0]
self.assertEqual("收口中", event["job_status_label"])
self.assertEqual("远端 Agent 已开始收集 Worker 日志", event["summary_text"])
self.assertEqual("worker_logs", event["focus_ref"]["focus_step_key"])
self.assertEqual("job-event:501", event["focus_ref"]["event_key"])
self.assertEqual("ops_job", event["source_focus_ref"]["kind"])
self.assertEqual("job-event:501", event["source_focus_ref"]["event_key"])
summary = payload["summary"]
self.assertEqual(1, summary["returned_total"])
self.assertEqual({"job_running": 1}, summary["event_type_counts"])
self.assertEqual({"running": 1}, summary["job_status_counts"])
self.assertEqual(10, summary["filters"]["limit"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,253 @@
import unittest
from unittest.mock import patch
from app.services.ops_policy_service import preview_ops_job_policy
class OpsPolicyServiceTests(unittest.TestCase):
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_node_bootstrap_preview_requires_control_plane(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "node.bootstrap",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "remote-agent",
"payload": {},
}
)
self.assertEqual("critical", preview["risk_level"])
self.assertTrue(preview["blocked"])
self.assertTrue(preview["approval_required"])
self.assertIn("node.bootstrap 仅支持 control-plane", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_batch_preview_summarizes_node_risks(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"status": "busy",
"current_load": 3,
"is_effective_worker": True,
"detect_participating": False,
},
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
},
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": ["mainland-controller-01"],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "batch",
"target_node_codes": ["mainland-controller-01", "mainland-worker-01"],
"execution_mode": "remote-agent",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
}
)
self.assertEqual("batch", preview["target_type"])
self.assertTrue(preview["blocked"])
self.assertTrue(preview["approval_required"])
self.assertEqual(2, preview["target_summary"]["nodes_total"])
self.assertEqual(1, preview["target_summary"]["blocked_nodes"])
self.assertEqual(1, preview["target_summary"]["approval_nodes"])
self.assertEqual(1, preview["target_summary"]["warning_nodes"])
self.assertEqual(2, len(preview["target_nodes"]))
controller_row = next(item for item in preview["target_nodes"] if item["node_code"] == "mainland-controller-01")
self.assertTrue(controller_row["blocked"])
self.assertTrue(controller_row["approval_required"])
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_preview_blocks_missing_artifact_and_wrong_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "local-runtime",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "",
},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("deploy.release 缺少 artifact_url", " ".join(preview["blocking_reasons"]))
self.assertIn("deploy.release 仅支持 remote-agent 或 ssh", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_preview_allows_ssh_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "ssh",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
}
)
self.assertFalse(any("仅支持 remote-agent 或 ssh" in item for item in preview["blocking_reasons"]))
self.assertTrue(preview["approval_required"])
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_service_status_preview_blocks_control_plane_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "service.status",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "control-plane",
"payload": {"service_name": "domaincheck-worker"},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("service.status 当前不支持 control-plane 执行方式", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_logs_collect_preview_blocks_local_runtime_on_remote_node(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "logs.collect",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "local-runtime",
"payload": {
"service_name": "domaincheck-worker",
"lines": 120,
},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("local-runtime 仅支持当前控制面本机节点", " ".join(preview["blocking_reasons"]))
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,89 @@
import io
import tarfile
import tempfile
import unittest
from pathlib import Path
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
class _BytesResponse:
def __init__(self, data: bytes) -> None:
self._data = data
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self, *args, **kwargs) -> bytes:
return self._data
def _build_release_archive() -> bytes:
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
content = b"hello-release"
info = tarfile.TarInfo(name="bundle/README.txt")
info.size = len(content)
archive.addfile(info, io.BytesIO(content))
return buffer.getvalue()
class OpsReleaseExecutorCoreTests(unittest.TestCase):
def test_execute_release_action_deploys_archive_and_switches_current(self) -> None:
archive_bytes = _build_release_archive()
commands: list[tuple[str, ...]] = []
def run_command(command: list[str], *, timeout: int = 60):
commands.append(tuple(command))
if command[:2] == ["systemctl", "restart"]:
return 0, "", ""
if command[:2] == ["systemctl", "is-active"]:
return 0, "active", ""
return 0, "", ""
with tempfile.TemporaryDirectory() as tmpdir:
ok, message, data = execute_release_action(
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/domaincheck.tar.gz",
"install_root": tmpdir,
"restart_services": ["domaincheck-worker"],
"health_check_services": ["domaincheck-worker"],
"health_check_urls": [],
"rollback_on_failure": "true",
"switch_current": "true",
},
run_command=run_command,
default_api_service_name="domaincheck-api",
urlopen_func=lambda request, timeout=120: _BytesResponse(archive_bytes),
user_agent="unit-test/0.1",
)
self.assertTrue(ok, message)
self.assertEqual("release deployed", message)
self.assertEqual("2026.04.18-rc1", data["release_version"])
current_link = Path(tmpdir) / "current"
self.assertTrue(current_link.is_symlink())
release_dir = Path(data["release_dir"])
self.assertTrue(release_dir.exists())
self.assertTrue((release_dir / ".release-meta.json").exists())
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
def test_build_remote_release_action_script_is_valid_python(self) -> None:
script = build_remote_release_action_script(
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/domaincheck.tar.gz",
}
)
compile(script, "<remote-release-script>", "exec")
self.assertIn("execute_release_action(", script)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,93 @@
import json
import unittest
from datetime import datetime
from app.services.ops_release_service import _serialize_release_row, _serialize_rollout_row
class OpsReleaseServiceContractTests(unittest.TestCase):
def test_serialize_release_row_adds_contract_aliases(self) -> None:
row = (
9,
"2026.04.18+bd6bcb2",
"stable",
"bd6bcb2",
"ready",
"https://example.com/domaincheck.tar.gz",
"sha256:test",
"release notes",
json.dumps(
{
"source": "package",
"manifest": {
"artifact_kind": "tarball",
"package_name": "domaincheck-2026.04.18+bd6bcb2.tgz",
"included_services": ["domaincheck-api", "domaincheck-worker"],
"rollback_hint": "切回上一个 current",
},
}
),
"www",
datetime(2026, 4, 18, 9, 0, 0),
datetime(2026, 4, 18, 9, 5, 0),
datetime(2026, 4, 18, 9, 6, 0),
)
payload = _serialize_release_row(row)
self.assertEqual("已就绪", payload["status_label"])
self.assertIn("已就绪", payload["summary_text"])
self.assertEqual("release_hub", payload["focus_ref"]["kind"])
self.assertEqual(9, payload["focus_ref"]["release_id"])
self.assertEqual("release_detail", payload["focus_ref"]["section"])
self.assertEqual("tarball", payload["manifest"]["artifact_kind"])
self.assertEqual("domaincheck-2026.04.18+bd6bcb2.tgz", payload["manifest"]["package_name"])
self.assertEqual(["domaincheck-api", "domaincheck-worker"], payload["manifest"]["included_services"])
def test_serialize_rollout_row_adds_contract_aliases(self) -> None:
row = (
11,
9,
"rollout-20260418090000-a1b2",
json.dumps({"region": "mainland", "role": "worker"}),
json.dumps(
{
"items": [
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
]
}
),
json.dumps({"execution_mode": "ssh", "batch_size": 1}),
"running",
1,
2,
2,
1,
json.dumps(
{
"status": "running",
"launched_targets": 1,
"remaining_targets": 1,
"active_jobs": 1,
"current_batch_index": 1,
"batches_total": 2,
}
),
"web-ui",
datetime(2026, 4, 18, 9, 10, 0),
datetime(2026, 4, 18, 9, 11, 0),
)
payload = _serialize_rollout_row(row)
self.assertEqual("执行中", payload["status_label"])
self.assertEqual(["mainland-worker-01", "mainland-worker-02"], payload["target_node_codes"])
self.assertIn("当前已覆盖 1/2 台节点", payload["summary_text"])
self.assertEqual("release_rollout", payload["focus_ref"]["kind"])
self.assertEqual(11, payload["focus_ref"]["rollout_id"])
self.assertEqual("rollout_jobs", payload["focus_ref"]["section"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,432 @@
import unittest
from unittest.mock import patch
from app.services.ops_release_service import (
build_release_execution_mode_recommendation,
build_release_rollout_gate,
build_rollout_target_operational_readiness,
)
class _EmptyCursor:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, query, params=None) -> None:
self.query = query
self.params = params
def fetchall(self):
return []
def fetchone(self):
return None
class _EmptyConnection:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return _EmptyCursor()
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
"recovery_decision": {
"action": "run_acceptance",
"label": "执行接管验收",
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
"window": "acceptance",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="ssh",
)
self.assertEqual("ssh", readiness["execution_mode"])
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
self.assertEqual([], readiness["blocking_reasons"])
self.assertTrue(readiness["rows"][0]["ssh_ready"])
self.assertTrue(readiness["rows"][0]["execution_ready"])
self.assertTrue(readiness["rows"][0]["agent_managed"])
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"metadata": {"current_release_version": "2026.04.11+1921318"},
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {"current_release_version": "2026.04.11+1921318"},
"last_seen_at": "2026-04-18 08:00:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
)
row = readiness["rows"][0]
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
self.assertEqual("release_hub", row["focus_ref"]["kind"])
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
self.assertEqual("noop", row["recovery_action"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.249",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
"summary": "已签发有效 Agent Token但节点尚未完成 register/heartbeat。",
"recovery_decision": {
"action": "bootstrap_run",
"label": "签发接入工单",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
"window": "bootstrap",
},
}
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("bootstrap_run", row["recovery_action"])
self.assertEqual("bootstrap_run", row["recommended_action_code"])
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
self.assertTrue(
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.250",
"ssh_user": "root",
"metadata": {
"delivery_queue": {
"state": "dead_letter",
"label": "死信 1",
"reason": "当前存在 1 条死信记录",
"pending_count": 0,
"dead_letter_count": 1,
"oldest_dead_letter_at": "2026-04-18 09:01:00",
}
},
"last_seen_at": "2026-04-18 09:05:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("dead_letter", row["delivery_queue_state"])
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
self.assertTrue(
any("死信" in item for item in row["blocking_reasons"])
)
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
self.assertTrue(
any("死信回执" in item for item in readiness["blocking_reasons"])
)
self.assertTrue(
any("死信队列" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "ssh",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口,当前不适合直接 SSH 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 11,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="ssh",
)
self.assertEqual("ssh", gate["execution_mode"])
self.assertEqual("SSH", gate["execution_mode_label"])
self.assertEqual("blocked", gate["status"])
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "remote-agent",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点 Agent 未在线,当前不适合直接 remote-agent 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 12,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="remote-agent",
)
self.assertEqual("remote-agent", gate["execution_mode"])
self.assertEqual("远端 Agent", gate["execution_mode_label"])
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_release_rollout_gate")
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
if execution_mode == "ssh":
return {
"status": "ready",
"status_label": "可发 Rollout",
"summary": "SSH 路径已收口",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"blocking_reasons": [],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
"rows": [],
},
}
return {
"status": "blocked",
"status_label": "存在阻断",
"summary": "remote-agent 尚未收口",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"blocking_reasons": ["Agent 未就绪"],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
"rows": [],
},
}
mock_build_gate.side_effect = _fake_gate
recommendation = build_release_execution_mode_recommendation(
{
"id": 18,
"release_version": "2026.04.18-rc2",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
)
self.assertEqual("ssh", recommendation["recommended_mode"])
self.assertEqual("SSH", recommendation["recommended_mode_label"])
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
self.assertIn("SSH", recommendation["reason"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,212 @@
import unittest
from app.services.ops_release_service import (
_build_release_launchpad_status,
_compact_latest_smart_rollout_preview,
)
class OpsReleaseServiceLaunchpadTests(unittest.TestCase):
def test_launchpad_status_blocks_when_package_missing(self) -> None:
status = _build_release_launchpad_status(
package={"available": False},
latest_package_preview={},
worker_preview={},
control_preview={},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("release_package", status["recommended_action_code"])
self.assertEqual(status["summary"], status["summary_text"])
self.assertEqual("release_hub", status["focus_ref"]["kind"])
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
def test_launchpad_status_blocks_when_rollout_preview_has_blockers(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"policy_preview": {"blocked": True}},
control_preview={"policy_preview": {"blocked": False}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("fix_rollout_blockers", status["recommended_action_code"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_blocks_when_final_signoff_missing(self) -> None:
status = _build_release_launchpad_status(
package={
"available": True,
"final_release_ok": False,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": False,
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("release_prepare", status["recommended_action_code"])
self.assertIn("final_release_report does not match latest_release: package_name", status["summary"])
self.assertIn("release-prepare", status["recommended_command"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_prefers_runtime_refresh_when_runtime_schema_is_stale(self) -> None:
status = _build_release_launchpad_status(
package={"available": True, "final_release_ok": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
runtime_build_info={
"runtime_schema_stale": True,
"route_surface_complete": True,
"route_surface_missing_keys": ["ops_node_handover_bootstrap_plan"],
},
)
self.assertEqual("attention", status["status"])
self.assertEqual("api-restart", status["recommended_action_code"])
self.assertEqual("待刷新运行时", status["status_label"])
self.assertTrue(status["runtime_refresh_required"])
self.assertTrue(status["runtime_schema_stale"])
self.assertIn("runtime-refresh-recover", status["recommended_command"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_marks_attention_when_confirmation_required(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"requires_confirmation": True, "policy_preview": {}},
control_preview={"policy_preview": {}},
)
self.assertEqual("attention", status["status"])
self.assertEqual("review_smart_rollout_preview", status["recommended_action_code"])
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
def test_launchpad_status_ready_when_worker_rollout_available(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True, "execution_mode": "ssh", "execution_mode_label": "SSH"}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("ready", status["status"])
self.assertEqual("publish_latest_worker", status["recommended_action_code"])
self.assertEqual("ssh", status["recommended_execution_mode"])
self.assertEqual("SSH", status["recommended_execution_mode_label"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_requires_managed_nodes_when_no_targets_available(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("attention", status["status"])
self.assertEqual("fix_managed_nodes", status["recommended_action_code"])
self.assertEqual("release_hub", status["focus_ref"]["kind"])
def test_launchpad_status_prefers_bootstrap_recovery_when_gap_row_has_recovery_action(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={
"smart_rollout": {"available": False},
"policy_preview": {
"blocked": True,
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-02",
"recovery_action": "bootstrap_run",
"recovery_label": "签发接入工单",
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
}
]
}
},
},
},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("bootstrap_run", status["recommended_action_code"])
self.assertEqual("mainland-worker-02", status["recommended_target_node_code"])
self.assertEqual("签发接入工单", status["recommended_recovery_label"])
self.assertEqual("当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。", status["recommended_recovery_summary"])
self.assertEqual(1, status["onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, status["onboarding_acceptance_ready_nodes"])
self.assertEqual(1, status["onboarding_gap_nodes_total"])
self.assertIn("node-bootstrap-run", status["recommended_command"])
def test_compact_latest_smart_rollout_preview_adds_rollout_preview_alias(self) -> None:
preview = _compact_latest_smart_rollout_preview(
"智能 Rollout 预检通过",
{
"release": {
"id": 18,
"release_version": "2026.04.18+bd6bcb2",
"channel": "stable",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
"package": {
"available": True,
"package_name": "domaincheck-2026.04.18+bd6bcb2",
"package_type": "tar.gz",
"sha256": "sha256:test",
"metadata": {
"manifest": {
"included_services": ["domaincheck-api", "domaincheck-worker"],
}
},
},
"smart_rollout": {
"available": True,
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"summary": "已按 Worker 预选 2 台节点。",
"target_selector": {"node_codes": ["mainland-worker-01", "mainland-worker-02"]},
"target_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"delivery_queue_state": "healthy",
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"delivery_queue_state": "retrying",
},
],
},
"preview": {
"target_summary": {"nodes_total": 2},
"batch_plan": {"batches_total": 2, "first_batch_size": 1},
"release_gate": {"status": "ready", "summary": "当前可推进。"},
},
},
)
self.assertEqual("remote-agent", preview["rollout_preview"]["execution_mode"])
self.assertEqual(2, preview["rollout_preview"]["target_nodes_total"])
self.assertEqual(2, preview["rollout_preview"]["expected_batches_total"])
self.assertEqual(2, preview["rollout_preview"]["expected_jobs_total"])
self.assertEqual("release_hub", preview["rollout_preview"]["focus_ref"]["kind"])
self.assertEqual("healthy", preview["smart_rollout"]["target_nodes"][0]["delivery_queue_state"])
self.assertEqual("retrying", preview["smart_rollout"]["target_nodes"][1]["delivery_queue_state"])
self.assertEqual("rollout_preview", preview["rollout_preview"]["focus_ref"]["section"])
self.assertEqual(["domaincheck-api", "domaincheck-worker"], preview["package"]["manifest"]["included_services"])

View File

@@ -0,0 +1,915 @@
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_release_service import (
build_smart_release_rollout_plan,
create_release_and_smart_rollout_from_latest_package,
create_release_from_latest_package,
create_smart_release_rollout,
get_latest_release_package_metadata,
get_release_package_archive_path,
get_release_package_sha256_path,
)
class OpsReleaseServicePackageMetadataTests(unittest.TestCase):
def test_get_latest_release_package_metadata_reads_latest_linux_package(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_20260418_120000"
staging_path = release_root / package_name
archive_path = release_root / f"{package_name}.tar.gz"
sha256_path = release_root / f"{package_name}.sha256.txt"
staging_path.mkdir(parents=True, exist_ok=True)
archive_path.write_bytes(b"linux-package")
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
(staging_path / "release_manifest.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"included": {
"docs": ["08_domainCheck_交付说明.md"],
"scripts": ["package_domain_release.sh", "verify_domain_release.sh"],
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"generated_at": "2026-04-18T12:00:00",
"staging_path": str(staging_path),
"archive_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "test-sha256",
"smoke_test_ok": True,
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(
{
"ok": True,
"latest_release": {
"package_name": package_name,
"archive_path": str(archive_path),
"sha256": "test-sha256",
},
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
data = get_latest_release_package_metadata()
self.assertTrue(data["available"])
self.assertEqual(package_name, data["package_name"])
self.assertEqual("tar.gz", data["package_type"])
self.assertTrue(data["archive_present"])
self.assertTrue(data["sha256_present"])
self.assertTrue(data["staging_present"])
self.assertEqual("test-sha256", data["sha256"])
self.assertEqual(f"/api/v1/ops/releases/packages/{package_name}/download", data["download_path"])
self.assertEqual(package_name, data["release_version_suggestion"])
self.assertIn("自检通过", data["notes_suggestion"])
self.assertTrue(data["final_release_ok"])
self.assertTrue(data["final_release_report_available"])
self.assertTrue(data["final_release_report_matches_latest"])
self.assertEqual("ready", data["final_release_gate_decision"])
self.assertEqual([], data["final_release_gate_blocked_reasons"])
self.assertEqual(package_name, data["metadata"]["manifest"]["package_name"])
def test_get_release_package_paths_support_metadata_fallback_for_zip(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_windows_demo"
package_dir = release_root / "exports"
package_dir.mkdir(parents=True, exist_ok=True)
archive_path = package_dir / f"{package_name}.zip"
sha256_path = package_dir / f"{package_name}.sha256.txt"
archive_path.write_bytes(b"zip-package")
sha256_path.write_text("sha256=zip-sha256", encoding="utf-8")
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "zip",
"generated_at": "2026-04-18T12:10:00",
"zip_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "zip-sha256",
"smoke_test_ok": False,
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
resolved_archive = get_release_package_archive_path(package_name)
resolved_sha256 = get_release_package_sha256_path(package_name)
metadata = get_latest_release_package_metadata()
self.assertEqual(archive_path, resolved_archive)
self.assertEqual(sha256_path, resolved_sha256)
self.assertEqual("zip", metadata["package_type"])
self.assertEqual(archive_path.name, metadata["archive_filename"])
self.assertFalse(metadata["smoke_test_ok"])
def test_get_latest_release_package_metadata_returns_unavailable_when_missing(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": tmpdir}):
data = get_latest_release_package_metadata()
self.assertFalse(data["available"])
self.assertIn("latest_release.json", data["reason"])
def test_get_latest_release_package_metadata_marks_stale_final_release_report(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_20260418_130000"
archive_path = release_root / f"{package_name}.tar.gz"
sha256_path = release_root / f"{package_name}.sha256.txt"
archive_path.write_bytes(b"linux-package")
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"archive_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "test-sha256",
"smoke_test_ok": True,
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(
{
"ok": True,
"latest_release": {
"package_name": "domaincheck_release_20260418_old",
"archive_path": str(release_root / "domaincheck_release_20260418_old.tar.gz"),
"sha256": "old-sha256",
},
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
data = get_latest_release_package_metadata()
self.assertTrue(data["available"])
self.assertFalse(data["final_release_ok"])
self.assertTrue(data["final_release_report_available"])
self.assertTrue(data["final_release_report_ok"])
self.assertFalse(data["final_release_report_matches_latest"])
self.assertIn("package_name", data["final_release_report_stale_reason"])
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_builds_payload(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_120000",
"release_version_suggestion": "domaincheck_release_20260418_120000",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"notes_suggestion": "导入自本机发布包 domaincheck_release_20260418_120000.tar.gz自检通过",
"smoke_test_ok": True,
"final_release_ok": True,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": True,
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_120000",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {}
mock_create_release.return_value = (
True,
"Release 已创建",
{
"release": {
"id": 7,
"release_version": "domaincheck_release_20260418_120000",
}
},
)
ok, message, data = create_release_from_latest_package(
{
"channel": "stable",
"status": "ready",
"created_by": "test-suite",
},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertTrue(ok)
self.assertEqual("Release 已创建", message)
mock_create_release.assert_called_once()
create_payload = mock_create_release.call_args.args[0]
self.assertEqual("domaincheck_release_20260418_120000", create_payload["release_version"])
self.assertEqual("bd6bcb2", create_payload["commit_sha"])
self.assertEqual(
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
create_payload["artifact_url"],
)
self.assertEqual("test-sha256", create_payload["checksum"])
self.assertEqual("latest_release_package", create_payload["metadata"]["created_from"])
self.assertEqual(
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
create_payload["metadata"]["package_source"]["download_url"],
)
self.assertFalse(data["deduplicated"])
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_is_idempotent(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_120000",
"release_version_suggestion": "domaincheck_release_20260418_120000",
"smoke_test_ok": True,
"final_release_ok": True,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": True,
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
}
mock_get_release_by_version.return_value = {
"id": 9,
"release_version": "domaincheck_release_20260418_120000",
}
ok, message, data = create_release_from_latest_package({}, control_plane_base_url="https://ops.example.com")
self.assertTrue(ok)
self.assertEqual("Release 已存在,按幂等返回", message)
self.assertTrue(data["deduplicated"])
self.assertEqual(9, data["release"]["id"])
mock_create_release.assert_not_called()
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_blocks_when_final_signoff_missing(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_130000",
"release_version_suggestion": "domaincheck_release_20260418_130000",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"smoke_test_ok": False,
"final_release_ok": False,
"final_release_report_available": False,
"final_release_report_ok": None,
"final_release_report_matches_latest": None,
"final_release_gate_decision": "",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_130000",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {}
ok, message, data = create_release_from_latest_package(
{
"channel": "stable",
"status": "ready",
"created_by": "test-suite",
},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
self.assertIn("smoke_test_failed_or_skipped", data["final_release_gate"]["blocked_reasons"])
self.assertIn("final_release_report_missing", data["final_release_gate"]["blocked_reasons"])
mock_create_release.assert_not_called()
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_blocks_when_final_report_is_stale(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_130001",
"release_version_suggestion": "domaincheck_release_20260418_130001",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"smoke_test_ok": True,
"final_release_ok": False,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": False,
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_130001",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {
"id": 19,
"release_version": "domaincheck_release_20260418_130001",
}
ok, message, data = create_release_from_latest_package(
{"created_by": "test-suite"},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
self.assertEqual(
"final_release_report does not match latest_release: package_name",
data["final_release_gate"]["stale_reason"],
)
self.assertIn("final_release_report_not_matching_latest", data["final_release_gate"]["blocked_reasons"])
self.assertEqual(19, data["release"]["id"])
mock_create_release.assert_not_called()
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_prefers_remote_ready_workers(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 1,
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
control_plan = build_smart_release_rollout_plan("control")
self.assertTrue(worker_plan["available"])
self.assertEqual("worker", worker_plan["mode"])
self.assertEqual(["mainland-worker-01"], worker_plan["target_selector"]["node_codes"])
self.assertEqual(["domaincheck-worker"], worker_plan["policy"]["deploy_payload"]["restart_services"])
self.assertTrue(control_plan["available"])
self.assertEqual("control", control_plan["mode"])
self.assertEqual(["mainland-controller-01"], control_plan["target_selector"]["node_codes"])
self.assertIn("domaincheck-api", control_plan["policy"]["deploy_payload"]["restart_services"])
self.assertEqual("remote-agent", worker_plan["policy"]["execution_mode"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_falls_back_to_ssh_ready_workers(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "busy",
"cluster_current_load": 1,
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertTrue(worker_plan["available"])
self.assertEqual("worker", worker_plan["mode"])
self.assertEqual("ssh", worker_plan["execution_mode"])
self.assertEqual("SSH", worker_plan["execution_mode_label"])
self.assertEqual(
["mainland-worker-01", "mainland-worker-02"],
worker_plan["target_selector"]["node_codes"],
)
self.assertEqual("ssh", worker_plan["policy"]["execution_mode"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_excludes_dead_letter_nodes_from_ready_candidates(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "dead_letter",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "healthy",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertTrue(worker_plan["available"])
self.assertEqual(["mainland-worker-02"], worker_plan["target_selector"]["node_codes"])
self.assertEqual("healthy", worker_plan["target_nodes"][0]["delivery_queue_state"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_returns_unavailable_when_only_dead_letter_nodes_exist(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "dead_letter",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertFalse(worker_plan["available"])
self.assertIn("死信回执", worker_plan["reason"])
self.assertEqual("dead_letter", worker_plan["target_nodes"][0]["delivery_queue_state"])
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_stops_when_latest_package_gate_blocks(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
False,
"最新发布包尚未完成最终签收,暂不允许创建 Release 或发起基于最新包的智能 Rollout。",
{
"package": {
"available": True,
"package_name": "domaincheck_release_20260418_blocked",
"final_release_ok": False,
},
"final_release_gate": {
"ready": False,
"status": "blocked",
"blocked_reasons": ["final_release_report_missing"],
},
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker"},
control_plane_base_url="https://ops.example.com",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
mock_list_managed_nodes_with_agent_state.assert_not_called()
mock_preview_ops_job_policy.assert_not_called()
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_requires_confirmation_before_rollout(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 21, "release_version": "domaincheck_release_20260418_120000"},
"deduplicated": False,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": True,
"warnings": ["本次 rollout 覆盖 control 节点,建议审批。"],
"release_gate": {"status_label": "待补巡检", "summary": "建议先确认控制面健康。"},
}
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "control"},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release 已创建,智能 Rollout 需要确认", message)
self.assertTrue(data["requires_confirmation"])
self.assertFalse(data["rollout_created"])
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_creates_rollout_after_confirmation(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 22, "release_version": "domaincheck_release_20260418_120000"},
"deduplicated": True,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 5, "release_id": 22, "rollout_code": "rollout-20260418-demo"},
"jobs": [],
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker", "confirm_risky": True},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["rollout_created"])
self.assertEqual(5, data["rollout"]["id"])
mock_create_release_rollout.assert_called_once()
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_uses_ssh_when_only_ssh_path_is_ready(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 23, "release_version": "domaincheck_release_20260418_ssh"},
"deduplicated": True,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 6, "release_id": 23, "rollout_code": "rollout-20260418-ssh"},
"jobs": [],
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker", "confirm_risky": True},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["rollout_created"])
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual("ssh", rollout_payload["policy"]["execution_mode"])
@patch("app.services.ops_release_service.get_release")
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_create_smart_release_rollout_for_existing_release_requires_confirmation(
self,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
mock_get_release,
) -> None:
mock_get_release.return_value = {
"id": 41,
"release_version": "domaincheck_release_existing",
"channel": "stable",
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": True,
"warnings": ["控制面节点建议审批。"],
"release_gate": {"status_label": "待审批", "summary": "需要审批。"},
}
ok, message, data = create_smart_release_rollout(41, {"mode": "control"})
self.assertTrue(ok)
self.assertEqual("智能 Rollout 需要确认", message)
self.assertFalse(data["release_created"])
self.assertTrue(data["requires_confirmation"])
self.assertFalse(data["rollout_created"])
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.get_release")
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_create_smart_release_rollout_for_existing_release_creates_rollout(
self,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
mock_get_release,
) -> None:
mock_get_release.return_value = {
"id": 42,
"release_version": "domaincheck_release_existing",
"channel": "stable",
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 8, "release_id": 42, "rollout_code": "rollout-20260418-existing"},
"jobs": [],
},
)
ok, message, data = create_smart_release_rollout(42, {"mode": "worker", "confirm_risky": True})
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertFalse(data["release_created"])
self.assertTrue(data["rollout_created"])
self.assertEqual(8, data["rollout"]["id"])
mock_create_release_rollout.assert_called_once()
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,118 @@
import subprocess
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_runtime_executor_service import execute_local_support_action, execute_ssh_action
class OpsRuntimeExecutorServiceTests(unittest.TestCase):
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_ssh_action_parses_structured_json_result(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["ssh"],
returncode=0,
stdout='login banner\n{"ok": true, "message": "health snapshot collected", "result": {"checks": {"api": {"state": "active"}}}}\n',
stderr="",
)
ok, message, data = execute_ssh_action(
{
"node_code": "mainland-worker-01",
"ssh_host": "121.204.244.248",
"ssh_port": 22,
"ssh_user": "root",
},
"health.snapshot",
{},
)
self.assertTrue(ok)
self.assertEqual("health snapshot collected", message)
self.assertEqual("active", data["checks"]["api"]["state"])
self.assertEqual("ssh", data["transport"]["executor"])
self.assertEqual("121.204.244.248", data["transport"]["ssh_host"])
issued_command = mock_run.call_args.args[0]
self.assertEqual("ssh", issued_command[0])
self.assertIn("root@121.204.244.248", issued_command)
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_collects_logs_with_payload(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["journalctl"],
returncode=0,
stdout="log line 1\nlog line 2",
stderr="",
)
ok, message, data = execute_local_support_action(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 180,
},
)
self.assertTrue(ok)
self.assertEqual("log line 1\nlog line 2", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual(180, data["lines"])
issued_command = mock_run.call_args.args[0]
self.assertEqual(
["journalctl", "-u", "domaincheck-worker", "-n", "180", "--no-pager"],
issued_command,
)
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_falls_back_to_worker_log_file(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["journalctl"],
returncode=1,
stdout="",
stderr="No journal files were opened due to insufficient permissions.",
)
with tempfile.TemporaryDirectory() as temp_dir:
log_path = Path(temp_dir) / "detect_worker.log"
log_path.write_text("fallback-1\nfallback-2\n", encoding="utf-8")
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
ok, message, data = execute_local_support_action(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 180,
},
)
self.assertTrue(ok)
self.assertEqual("fallback-1\nfallback-2", message)
self.assertEqual("file_fallback", data["collection_source"])
self.assertTrue(data["fallback_used"])
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_restarts_service(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["systemctl"],
returncode=0,
stdout="",
stderr="",
)
ok, message, data = execute_local_support_action(
"service.restart",
{
"service_name": "domaincheck-worker",
},
)
self.assertTrue(ok)
self.assertEqual("domaincheck-worker restart completed", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual("restart", data["systemctl_action"])
issued_command = mock_run.call_args.args[0]
self.assertEqual(["systemctl", "restart", "domaincheck-worker"], issued_command)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,75 @@
import unittest
from unittest.mock import MagicMock, patch
import app.services.ops_agent_service as ops_agent_service
import app.services.ops_job_service as ops_job_service
class OpsSchemaInitTests(unittest.TestCase):
@patch("app.services.ops_job_service.get_db")
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_job_service._OPS_SCHEMA_READY
ops_job_service._OPS_SCHEMA_READY = False
try:
ops_job_service.ensure_ops_schema()
ops_job_service.ensure_ops_schema()
finally:
ops_job_service._OPS_SCHEMA_READY = previous_ready
self.assertEqual(1, mock_get_db.call_count)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(ops_job_service._OPS_SCHEMA_ADVISORY_LOCK_KEY,),
)
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
conn.commit.assert_called_once()
@patch("app.services.ops_agent_service.ensure_ops_schema")
@patch("app.services.ops_agent_service.get_db")
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
self,
mock_get_db,
mock_ensure_ops_schema,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
try:
ops_agent_service.ensure_ops_agent_schema()
ops_agent_service.ensure_ops_agent_schema()
finally:
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
self.assertEqual(2, mock_ensure_ops_schema.call_count)
self.assertEqual(1, mock_get_db.call_count)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(ops_agent_service._OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,),
)
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
conn.commit.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,607 @@
import unittest
from typing import Optional
from unittest.mock import patch
from app.services.ops_service import get_ops_activity_stream
class OpsServiceActivityTests(unittest.TestCase):
@staticmethod
def _runtime_status(detect: Optional[dict] = None) -> dict:
return {"detect": dict(detect or {})}
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_uses_job_detail_for_bootstrap_jobs(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_get_ops_runbook.return_value = {"control_sequences": []}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"agent_state": "pending_bootstrap",
"agent_state_label": "待接入",
}
]
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 51,
"job_code": "ops-bootstrap-51",
"action": "node.bootstrap",
"status": "success",
"target_node_code": "mainland-worker-02",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
"result": {
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
}
},
}
]
mock_list_release_rollouts.return_value = []
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("节点接入工单", item["title"])
self.assertEqual("job_detail", item["ui_intent"]["kind"])
self.assertEqual("job_detail", item["ui_intent_kind"])
self.assertEqual("控制面", item["execution_mode_label"])
self.assertEqual("成功", item["status_label"])
self.assertIn("尚未回连 Agent", item["summary"])
self.assertEqual(item["summary"], item["summary_text"])
self.assertIn("方式 控制面", item["meta_text"])
self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"])
self.assertEqual(["mainland-worker-02"], item["target_node_codes"])
self.assertEqual(51, item["focus_ref"]["job_id"])
self.assertEqual("node.bootstrap", item["focus_ref"]["action"])
self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_runbook_sequences_after_live_events(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-001",
"playbook_title": "标准巡检",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
}
]
}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
"primary_action_code": "fix_managed_nodes",
"primary_resolution": {
"ok": True,
"driver_action_code": "fix_managed_nodes",
"driver_action_label": "补执行面接管",
"driver_node_codes": ["mainland-controller-01"],
"message": "当前建议先补执行面接管。",
"resolved_at": "2026-04-18 09:12:00",
},
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("playbook_run", payload["items"][0]["kind"])
self.assertEqual("runbook_sequence", payload["items"][1]["kind"])
runbook_item = payload["items"][1]
self.assertEqual("release_progression", runbook_item["sequence_key"])
self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"])
self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"])
self.assertEqual("attention", runbook_item["status"])
self.assertIn("当前建议动作:补执行面接管", runbook_item["summary"])
self.assertIn("release_progression", runbook_item["meta_text"])
self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"group_title": "执行现场",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 2,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("SSH", payload["items"][0]["execution_mode_label"])
self.assertIn("执行 SSH", payload["items"][0]["meta_text"])
self.assertEqual("SSH", payload["items"][1]["execution_mode_label"])
self.assertIn("方式 SSH", payload["items"][1]["summary"])
self.assertIn("执行 SSH", payload["items"][1]["meta_text"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_prefers_job_result_summary_and_focus_ref(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:20:00",
"created_at": "2026-04-18 09:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
"event_key": "job-event:888",
"step_key": "worker_logs",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", item["summary"])
self.assertEqual("ops_job", item["focus_ref"]["kind"])
self.assertEqual("job-event:888", item["focus_ref"]["event_key"])
self.assertEqual("worker_logs", item["focus_ref"]["step_key"])
self.assertIn("执行 远端 Agent", item["meta_text"])
self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"status": "running",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 10:20:00",
"created_at": "2026-04-18 10:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "节点端已经收到采集任务。",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("attention", item["status"])
self.assertEqual("running", item["job_status"])
self.assertEqual("failed_local", item["start_delivery_state"])
self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"])
self.assertIn("开始回执未成功送达控制面", item["summary"])
self.assertIn("temporary offline", item["summary"])
self.assertIn("开始回执异常", item["meta_text"])
self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"])
self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"])
self.assertEqual("attention", payload["summary"]["status"])
self.assertEqual("待处理", payload["summary"]["status_label"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_label_for_rollout(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = [
{
"id": 201,
"release_id": 18,
"rollout_code": "rollout-20260418093000-a1b2",
"policy": {
"execution_mode": "ssh",
},
"status": "running",
"status_label": "执行中",
"target_nodes": [{"node_code": "mainland-worker-01"}],
"jobs_total": 2,
"jobs_created": 1,
"batch_cursor": 1,
"batches_total": 2,
"result_summary": {},
"summary_text": "当前已覆盖 1/1 台节点,仍有 1 条任务执行中。",
"focus_ref": {
"kind": "release_rollout",
"rollout_id": 201,
"rollout_code": "rollout-20260418093000-a1b2",
"release_id": 18,
"section": "rollout_jobs",
},
"created_by": "web-ui",
"created_at": "2026-04-18 09:30:00",
"updated_at": "2026-04-18 09:31:00",
}
]
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("rollout", item["kind"])
self.assertEqual("SSH", item["execution_mode_label"])
self.assertIn("执行 SSH", item["meta_text"])
self.assertEqual("执行中", item["status_label"])
self.assertIn("当前已覆盖 1/1 台节点", item["summary"])
self.assertEqual("release_rollout", item["focus_ref"]["kind"])
self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_supports_execution_mode_filter(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 1,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
},
{
"id": 62,
"job_code": "ops-agent-62",
"action": "health.snapshot",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-02",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:08:00",
"created_at": "2026-04-18 09:07:00",
},
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh")
self.assertEqual(2, len(payload["items"]))
self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"]))
self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"])
self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"])
self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_execution_scene_and_log_sync_items(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
detect = {
"active_job": {
"job_code": "detect-001",
"latest_cycle_event": {
"created_at": "2026-04-18 09:12:30",
},
},
"phase_label": "运行中",
"phase_detail": "代理池正常,现场可继续观察",
"recent_event": "域名检测完成: example.com",
"recent_warning": "",
"log_sync": {
"enabled": True,
"mode": "key",
"mode_label": "关键回传",
"description": "当前为关键回传;适合常态联调。",
"line_count": 12,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"last_at": "2026-04-18 09:12:31",
"preview_lines": ["worker log line"],
},
"participation_summary": {
"summary": "当前参与检测节点 1 台,其中执行/已领 1 台,近窗刚有吞吐 0 台。",
"effective_online_nodes": 1,
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"last_heartbeat_at": "2026-04-18 09:12:29",
"participation_state": "dispatch_active",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(detect),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
items_by_kind = {item["kind"]: item for item in payload["items"]}
self.assertIn("execution_scene", items_by_kind)
self.assertIn("log_sync", items_by_kind)
execution_scene_item = items_by_kind["execution_scene"]
log_sync_item = items_by_kind["log_sync"]
self.assertEqual("running", execution_scene_item["status"])
self.assertEqual("执行中", execution_scene_item["status_label"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"])
self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"])
self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"])
self.assertIn("执行/已领 1", execution_scene_item["meta_text"])
self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"])
self.assertEqual("running", log_sync_item["status"])
self.assertEqual("执行中", log_sync_item["status_label"])
self.assertEqual("control-plane", log_sync_item["execution_mode"])
self.assertEqual("控制面", log_sync_item["execution_mode_label"])
self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"])
self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"])
self.assertIn("关键日志观察模式", log_sync_item["summary"])
self.assertEqual("running", payload["summary"]["status"])
self.assertEqual("执行中", payload["summary"]["status_label"])
self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"])
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,925 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_codex_action, resolve_codex_action
def _build_codex_brief_entry(
*,
key: str,
executor_kind: str,
primary_action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
target_method: str = "POST",
sequence_key: str = "",
primary_action_payload: dict | None = None,
secondary_action_code: str = "",
secondary_action_payload: dict | None = None,
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
primary_action_payload = dict(primary_action_payload or {})
secondary_action_payload = dict(secondary_action_payload or {})
normalized_focus_ref = dict(focus_ref or {})
normalized_primary_focus_ref = dict(primary_focus_ref or {})
normalized_secondary_focus_ref = dict(secondary_focus_ref or {})
if normalized_primary_focus_ref and not dict(primary_action_payload.get("focus_ref") or {}):
primary_action_payload["focus_ref"] = dict(normalized_primary_focus_ref)
if normalized_secondary_focus_ref and not dict(secondary_action_payload.get("focus_ref") or {}):
secondary_action_payload["focus_ref"] = dict(normalized_secondary_focus_ref)
request_payload_preview = (
{
"sequence_key": sequence_key,
"secondary": False,
"action_payload": primary_action_payload,
}
if executor_kind == "runbook_sequence"
else {
"action_code": primary_action_code,
"node_codes": ["mainland-controller-01"],
"action_payload": primary_action_payload,
}
)
return {
"key": key,
"title": "测试动作",
"summary": "测试摘要",
"reason": "测试原因",
"meta_text": "meta",
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"focus_ref": normalized_focus_ref,
"primary_focus_ref": normalized_primary_focus_ref,
"secondary_focus_ref": normalized_secondary_focus_ref,
"primary_label": "主动作",
"secondary_label": "次动作" if secondary_action_code else "",
"primary_action_code": primary_action_code,
"secondary_action_code": secondary_action_code,
"autopilot": {
"status_text": "测试状态",
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"backend_handled": True,
"risk_level": "low" if recommendation == "auto_execute" else "high",
"reason": "autopilot reason",
"target_method": target_method,
"target_api": target_api,
},
"primary_execution": {
"label": "主动作",
"node_codes": ["mainland-controller-01"],
"action_payload": primary_action_payload,
"request_payload_preview": request_payload_preview,
},
"secondary_execution": (
{
"label": "次动作",
"node_codes": [],
"action_payload": secondary_action_payload,
}
if secondary_action_code
else {}
),
}
def _build_preview_data(
*,
executor_kind: str,
action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
request_payload: dict,
sequence_key: str = "",
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
return {
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"execution_chain": "ops-driver -> preview",
"focus_ref": dict(focus_ref or {}),
"primary_focus_ref": dict(primary_focus_ref or {}),
"secondary_focus_ref": dict(secondary_focus_ref or {}),
"primary": {
"label": "主动作",
"action_code": action_code,
"target_api": target_api,
"target_method": "POST",
"request_payload": dict(request_payload or {}),
"focus_ref": dict(primary_focus_ref or {}),
"execution_profile": {
"action_code": action_code,
"backend_handled": True,
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if recommendation == "auto_execute" else "high",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
},
"secondary": {},
}
def _build_codex_activity_focus_entry(
*,
key: str,
activity_key: str,
activity_kind: str,
ui_intent_kind: str,
focus_ref: dict | None = None,
source_focus_ref: dict | None = None,
target_node_codes: list[str] | None = None,
) -> dict:
normalized_focus_ref = dict(focus_ref or {})
return {
"key": key,
"activity_key": activity_key,
"activity_kind": activity_kind,
"title": "观察焦点",
"subtitle": "日志回传待确认",
"summary": "该焦点属于观察项,应进入对应工作区查看上下文。",
"status": "attention",
"status_label": "待关注",
"meta_text": "activity-meta",
"occurred_at": "2026-04-18T06:00:00",
"target_node_codes": list(target_node_codes or []),
"focus_ref": normalized_focus_ref,
"source_focus_ref": dict(source_focus_ref or {}),
"ui_intent": {
"kind": ui_intent_kind,
"activity_key": activity_key,
},
"ui_intent_kind": ui_intent_kind,
"focus_action_code": "focus_activity_item",
"focus_action_payload": {
"activity_key": activity_key,
"kind": activity_kind,
"status": "attention",
"ui_intent": {
"kind": ui_intent_kind,
"activity_key": activity_key,
},
"focus_ref": normalized_focus_ref,
},
"observation_only": True,
}
class OpsServiceCodexActionTests(unittest.TestCase):
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_returns_gate_for_safe_auto(self, mock_get_ops_codex_brief, mock_preview_driver_action) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "recommendation:enable-log-sync"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="recommendation:enable-log-sync",
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
),
)
ok, message, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("Codex 驾驶动作解析完成", message)
self.assertEqual("recommendation:enable-log-sync", data["entry_key"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_requires_confirmation_for_guarded_runbook(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
secondary_action_code="focus_release_hub",
secondary_action_payload={"release_id": 7},
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
),
)
ok, _, data = resolve_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
}
)
self.assertTrue(ok)
self.assertEqual("confirmation_required", data["gate"]["decision"])
self.assertFalse(data["gate"]["will_execute"])
self.assertTrue(data["gate"]["confirm_required"])
self.assertEqual("release_progression", data["gate"]["sequence_key"])
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
self.assertEqual("worker", data["focus_ref"]["mode"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_can_select_secondary_section(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
secondary_action_code="focus_release_hub",
secondary_action_payload={"release_id": 7},
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
],
}
preview_data = _build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
preview_data["secondary"] = {
"label": "查看版本区",
"action_code": "focus_release_hub",
"target_api": "/api/v1/ops/runbook/sequences/release_progression/execute",
"target_method": "POST",
"request_payload": {
"secondary": True,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"execution_profile": {
"action_code": "focus_release_hub",
"backend_handled": False,
"automation_level": "safe_auto",
"recommendation": "auto_execute",
"executor_mode_hint": "ui-intent",
"risk_level": "low",
"confirm_required": False,
"ui_only": False,
"blocked": False,
"requires_fields": [],
"missing_fields": [],
"reason": "secondary preview reason",
},
}
mock_preview_driver_action.return_value = (True, "ok", preview_data)
ok, _, data = resolve_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
"secondary": True,
}
)
self.assertTrue(ok)
self.assertEqual("secondary", data["selected_section"])
self.assertTrue(data["gate"]["secondary"])
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
self.assertEqual("auto_execute", data["gate"]["recommendation"])
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
self.assertEqual("release_detail", data["focus_ref"]["section"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_can_fall_back_to_activity_focus_entry(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
_build_codex_activity_focus_entry(
key="activity:ops-job-42",
activity_key="ops-job-42",
activity_kind="ops_job",
ui_intent_kind="job_events",
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
source_focus_ref={
"kind": "ops_job_event",
"job_key": "job-42",
"event_key": "event-1",
},
target_node_codes=["mainland-controller-01"],
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_activity_item",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_activity_item",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"activity_key": "ops-job-42",
"kind": "ops_job",
"status": "attention",
"ui_intent": {
"kind": "job_events",
"activity_key": "ops-job-42",
},
"focus_ref": {
"kind": "ops_job",
"job_key": "job-42",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
primary_focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
),
)
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("activity:ops-job-42", data["entry_key"])
self.assertTrue(data["selected_entry"]["observation_only"])
self.assertEqual("ops_job", data["selected_entry"]["activity_kind"])
self.assertEqual("focus_activity_item", data["preview_request"]["primary_action_code"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("job-42", data["focus_ref"]["job_key"])
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_safe_auto_driver_action(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "recommendation:enable-log-sync"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="recommendation:enable-log-sync",
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
)
],
}
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/codex-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload=request_payload,
),
)
mock_execute_driver_action.return_value = (
True,
"执行成功",
{"handled": True, "action_code": "enable_log_sync_key"},
)
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("执行成功", message)
self.assertTrue(data["executed"])
mock_execute_driver_action.assert_called_once_with(request_payload)
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_blocks_guarded_runbook_without_confirm(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
),
)
ok, message, data = execute_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
}
)
self.assertFalse(ok)
self.assertIn("确认", message)
self.assertFalse(data["executed"])
mock_execute_ops_runbook_sequence.assert_not_called()
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_activity_focus_as_safe_auto(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
_build_codex_activity_focus_entry(
key="activity:ops-job-42",
activity_key="ops-job-42",
activity_kind="ops_job",
ui_intent_kind="job_events",
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_activity_item",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_activity_item",
"node_codes": [],
"action_payload": {
"activity_key": "ops-job-42",
"kind": "ops_job",
"status": "attention",
"ui_intent": {
"kind": "job_events",
"activity_key": "ops-job-42",
},
"focus_ref": {
"kind": "ops_job",
"job_key": "job-42",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
primary_focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
),
)
mock_execute_driver_action.return_value = (
True,
"执行成功",
{"handled": True, "action_code": "focus_activity_item"},
)
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("执行成功", message)
self.assertTrue(data["executed"])
mock_execute_driver_action.assert_called_once()
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_supports_focus_latest_job_events_without_ui_intent(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-81", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
{
"key": "activity:ops-job-81",
"activity_key": "ops-job-81",
"activity_kind": "ops_job",
"title": "开始回执异常",
"subtitle": "ops-agent-81",
"summary": "节点已接单执行,但开始回执未成功送达控制面。",
"status": "attention",
"status_label": "待处理",
"meta_text": "ops-agent-81",
"occurred_at": "2026-04-18T06:00:00",
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"ui_intent": {},
"ui_intent_kind": "",
"focus_action_code": "focus_latest_job_events",
"focus_action_payload": {
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
"observation_only": True,
}
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_latest_job_events",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_latest_job_events",
"node_codes": ["mainland-worker-01"],
"action_payload": {
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
},
primary_focus_ref={
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
},
),
)
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("activity:ops-job-81", data["entry_key"])
self.assertEqual("focus_latest_job_events", data["preview_request"]["primary_action_code"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_guarded_runbook_after_confirm(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
)
],
}
request_payload = {
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload=request_payload,
sequence_key="release_progression",
),
)
mock_execute_ops_runbook_sequence.return_value = (
True,
"runbook 执行成功",
{"sequence_key": "release_progression", "driver_action_code": "create_release_rollout_worker"},
)
ok, message, data = execute_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
"confirm": True,
}
)
self.assertTrue(ok)
self.assertEqual("runbook 执行成功", message)
self.assertTrue(data["executed"])
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,497 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_codex_brief
class OpsServiceCodexBriefTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_classifies_safe_ui_and_runbook_entries(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前建议已形成。",
"go_live_summary": {
"go_live_status": "attention",
"publish_ready": False,
"publish_status": "attention",
"publish_status_label": "可发布但建议先复核",
"operator_title": "先补现场日志回传",
"next_step_action_code": "enable_log_sync_key",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 1,
},
"activity_focus": [],
"entries": [
{
"key": "recommendation:enable-scene-log-sync",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先补现场日志回传",
"summary": "优先打开关键日志回传。",
"reason": "当前还看不到现场过程日志。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "danger",
"meta_text": "",
"detail_lines": ["参与节点 2 台"],
"node_codes": ["mainland-worker-01"],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"secondary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs.full",
},
"primary_label": "开启关键回传",
"secondary_label": "开启全量回传",
"primary_action_code": "enable_log_sync_key",
"secondary_action_code": "enable_log_sync_full",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"secondary_action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs.full",
}
},
},
{
"key": "recommendation:release-missing",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先建立 Release",
"summary": "当前还没有默认 Release。",
"reason": "需要先进入版本区。",
"status": "planned",
"level_label": "待补齐",
"tag_type": "info",
"meta_text": "",
"detail_lines": [],
"node_codes": [],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "创建 Release",
"secondary_label": "",
"primary_action_code": "open_release_dialog",
"secondary_action_code": "",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {},
"secondary_action_payload": {},
},
{
"key": "runbook:release_progression",
"kind": "runbook_sequence",
"lane": "standard_path",
"lane_label": "标准路径",
"title": "版本发布与放量",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"status": "attention",
"level_label": "待处理",
"tag_type": "warning",
"meta_text": "Release / Rollout",
"detail_lines": ["当前解析Worker 灰度"],
"node_codes": ["mainland-controller-01"],
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"primary_label": "Worker 灰度",
"secondary_label": "查看版本区",
"primary_action_code": "create_release_rollout_worker",
"secondary_action_code": "focus_release_hub",
"primary_node_codes": ["mainland-controller-01"],
"secondary_node_codes": [],
"primary_action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
"secondary_action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
"occurred_at": "2026-04-18T06:00:00",
},
],
}
payload = get_ops_codex_brief()
self.assertEqual(3, payload["summary"]["total"])
self.assertEqual("attention", payload["go_live_summary"]["go_live_status"])
self.assertEqual("attention", payload["summary"]["go_live_status"])
self.assertEqual("先补现场日志回传", payload["summary"]["operator_title"])
self.assertEqual("attention", payload["summary"]["publish_status"])
self.assertFalse(payload["summary"]["publish_ready"])
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["summary"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(0, payload["summary"]["activity_focus_total"])
self.assertEqual("当前建议已形成。", payload["driver_feed"]["headline"])
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
self.assertEqual(2, payload["summary"]["auto_execute_total"])
self.assertEqual(1, payload["summary"]["confirm_then_execute_total"])
self.assertEqual(0, payload["summary"]["open_ui_total"])
self.assertEqual(1, payload["summary"]["preview_only_total"])
self.assertEqual(3, payload["summary"]["backend_handled_total"])
self.assertEqual("attention", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("recommendation:enable-scene-log-sync", payload["focus"]["entry_key"])
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", payload["focus"]["primary_focus_ref"]["scene_key"])
self.assertEqual("auto_execute", payload["entries"][0]["autopilot"]["recommendation"])
self.assertIn("ops_observability_contract", payload["entries"][0]["contract_keys"])
self.assertIn("ops_stack_diagnosis_contract", payload["entries"][0]["contract_keys"])
self.assertEqual("scene.logs", payload["entries"][0]["primary_focus_ref"]["scene_key"])
self.assertEqual("safe_auto", payload["entries"][1]["autopilot"]["automation_level"])
self.assertIn("release_hub_contract", payload["entries"][1]["contract_keys"])
runbook_entry = payload["entries"][2]
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
self.assertEqual("confirm_then_execute", runbook_entry["autopilot"]["recommendation"])
self.assertIn("ops_playbook_contract", runbook_entry["contract_keys"])
self.assertIn("release_hub_contract", runbook_entry["contract_keys"])
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
self.assertEqual("worker", runbook_entry["primary_focus_ref"]["mode"])
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
self.assertEqual(
"/api/v1/ops/runbook/sequences/release_progression/execute",
runbook_entry["autopilot"]["target_api"],
)
self.assertEqual(
{
"sequence_key": "release_progression",
"secondary": False,
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
runbook_entry["primary_execution"]["request_payload_preview"],
)
self.assertEqual(
{
"sequence_key": "release_progression",
"secondary": True,
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
runbook_entry["secondary_execution"]["request_payload_preview"],
)
self.assertEqual("attention", payload["automation_coverage"]["launch_status"])
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["guarded_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_marks_mixed_and_blocked_entries(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前建议需要先复核。",
"go_live_summary": {
"go_live_status": "blocked",
"publish_ready": False,
"publish_status": "blocked",
"publish_status_label": "暂不可发布",
"operator_title": "先补执行面接管",
"next_step_action_code": "fix_managed_nodes",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "补执行面接管",
"launchpad_recommended_recovery_summary": "先处理 launchpad 缺口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"activity_focus": [],
"entries": [
{
"key": "recommendation:release-launchpad-fix-managed",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先补执行面接管",
"summary": "先处理 launchpad 缺口。",
"reason": "当前动作可能直接 sync也可能转为补接入。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "danger",
"meta_text": "",
"detail_lines": [],
"node_codes": ["mainland-controller-01"],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 9,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 9,
"section": "default_rollout_gate",
},
"primary_label": "补执行面接管",
"secondary_label": "",
"primary_action_code": "fix_managed_nodes",
"secondary_action_code": "",
"primary_node_codes": ["mainland-controller-01"],
"secondary_node_codes": [],
"primary_action_payload": {"channel": "stable"},
"secondary_action_payload": {},
},
{
"key": "recommendation:inspect-participating",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先看执行现场",
"summary": "先跑参与节点巡检。",
"reason": "缺少目标节点时不应误判为可自动执行。",
"status": "attention",
"level_label": "推荐",
"tag_type": "warning",
"meta_text": "",
"detail_lines": [],
"node_codes": [],
"executor_kind": "driver_action",
"primary_label": "执行标准巡检",
"secondary_label": "",
"primary_action_code": "run_inspection_participating",
"secondary_action_code": "",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {},
"secondary_action_payload": {},
},
],
}
payload = get_ops_codex_brief()
self.assertEqual("blocked", payload["summary"]["publish_status"])
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("补执行面接管", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(1, payload["summary"]["resolve_first_total"])
self.assertEqual(1, payload["summary"]["blocked_total"])
self.assertEqual("blocked", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("mixed", payload["entries"][0]["autopilot"]["automation_level"])
self.assertEqual("resolve_first", payload["entries"][0]["autopilot"]["recommendation"])
self.assertEqual("default_rollout_gate", payload["entries"][0]["primary_focus_ref"]["section"])
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
self.assertEqual(1, payload["automation_coverage"]["blocked_total"])
self.assertEqual(2, payload["automation_coverage"]["human_dependency_total"])
blocked_entry = payload["entries"][1]
self.assertTrue(blocked_entry["autopilot"]["blocked"])
self.assertEqual(["node_codes"], blocked_entry["primary_execution"]["missing_fields"])
self.assertEqual("blocked", blocked_entry["autopilot"]["recommendation"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_exposes_activity_focus_and_uses_it_as_fallback_focus(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前先盯住执行现场。",
"scene_log_observation": {},
"entries": [],
"activity_focus": [
{
"key": "activity:execution-scene:live",
"activity_key": "execution-scene:live",
"activity_kind": "execution_scene",
"title": "当前执行现场",
"subtitle": "运行中",
"summary": "当前参与检测节点 2 台,建议先看现场与日志回传。",
"status": "running",
"status_label": "执行中",
"meta_text": "执行/已领 2 / 待命 1",
"occurred_at": "2026-04-18 09:30:00",
"target_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"source_focus_ref": {},
"ui_intent": {
"kind": "focus_execution_scene",
"open_log_drawer": True,
},
"ui_intent_kind": "focus_execution_scene",
"focus_action_code": "focus_activity_item",
"focus_action_payload": {
"activity_key": "execution-scene:live",
"kind": "execution_scene",
"status": "running",
"ui_intent": {
"kind": "focus_execution_scene",
"open_log_drawer": True,
},
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
},
"observation_only": True,
}
],
}
payload = get_ops_codex_brief()
self.assertEqual(0, payload["summary"]["total"])
self.assertEqual(1, payload["summary"]["activity_focus_total"])
self.assertEqual("activity:execution-scene:live", payload["focus"]["entry_key"])
self.assertEqual("open_ui", payload["focus"]["recommendation"])
self.assertIn("ops_observability_contract", payload["focus"]["contract_keys"])
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
self.assertEqual(1, len(payload["activity_focus"]))
self.assertEqual("focus_execution_scene", payload["activity_focus"][0]["ui_intent_kind"])
self.assertEqual(
"ops_driver_contract",
payload["activity_focus"][0]["contract_navigation"]["primary_contract_key"],
)
self.assertTrue(payload["activity_focus"][0]["observation_only"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_can_fallback_to_scene_log_observation_focus(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "先看现场日志。",
"entries": [],
"activity_focus": [],
"scene_log_observation": {
"status": "waiting_sample",
"status_label": "等待样本",
"summary": "日志回传已开启,但样本还没形成。",
"focus_ref": {
"kind": "node_scene_log",
"node_code": "mainland-worker-01",
"mode": "key",
"limit": 120,
},
"contract_keys": ["ops_observability_contract", "ops_stack_diagnosis_contract"],
},
}
payload = get_ops_codex_brief()
self.assertEqual("scene-log-observation", payload["focus"]["entry_key"])
self.assertEqual("open_ui", payload["focus"]["recommendation"])
self.assertEqual("node_scene_log", payload["focus"]["focus_ref"]["kind"])
self.assertEqual("mainland-worker-01", payload["focus"]["focus_ref"]["node_code"])
self.assertEqual("waiting_sample", payload["scene_log_observation"]["status"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_marks_bootstrap_run_as_agent_protocol_flow(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "先执行节点接入收口。",
"activity_focus": [],
"entries": [
{
"key": "priority:handover-gap-bootstrap_run",
"kind": "priority_recommendation",
"lane": "do_now",
"lane_label": "当前优先",
"title": "先补接管缺口",
"summary": "Launchpad 已明确给出首台节点的接入收口动作。",
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "warning",
"meta_text": "overview.recommendation",
"detail_lines": ["当前已收敛到单节点接入动作。"],
"node_codes": ["mainland-worker-01"],
"executor_kind": "driver_action",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01", "section": "bootstrap_plan"},
"primary_label": "跑接入收口",
"secondary_label": "查看 Bootstrap 计划",
"primary_action_code": "bootstrap_run",
"secondary_action_code": "create_bootstrap_plan",
"primary_node_codes": ["mainland-worker-01"],
"secondary_node_codes": ["mainland-worker-01"],
"primary_action_payload": {"node_code": "mainland-worker-01"},
"secondary_action_payload": {"node_code": "mainland-worker-01"},
}
],
}
payload = get_ops_codex_brief()
self.assertEqual("priority:handover-gap-bootstrap_run", payload["focus"]["entry_key"])
self.assertEqual("resolve_first", payload["focus"]["recommendation"])
self.assertIn("ops_agent_protocol", payload["entries"][0]["contract_keys"])
self.assertEqual("bootstrap_run", payload["entries"][0]["primary_action_code"])
self.assertEqual("mainland-worker-01", payload["entries"][0]["primary_execution"]["node_codes"][0])
self.assertEqual("/api/v1/ops/driver-actions/execute", payload["entries"][0]["autopilot"]["target_api"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,295 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import (
get_ops_blueprint,
get_ops_capabilities,
get_ops_contract_detail,
get_ops_contract_registry,
get_ops_runbook,
)
class OpsServiceContractRegistryTests(unittest.TestCase):
def test_contract_registry_lists_all_frozen_ops_contracts(self) -> None:
payload = get_ops_contract_registry()
self.assertEqual("2026-04-18", payload["registry_version"])
self.assertEqual("v1", payload["schema_version"])
self.assertEqual(11, payload["contracts_total"])
self.assertEqual(
[
"ops_job_contract",
"ops_agent_protocol",
"release_hub_contract",
"ops_driver_contract",
"ops_playbook_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
"ops_doctor_decision_contract",
"ops_go_live_signoff_contract",
"ops_go_live_bundle_contract",
"ops_go_live_review_contract",
],
[item["key"] for item in payload["contracts"]],
)
self.assertIn("/api/v1/ops/contracts/{contract_key}", payload["discovery_entrypoints"])
self.assertEqual(
"docs/schemas/ops_job_contract.md",
payload["contracts_by_key"]["ops_job_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/jobs/{job_id}/dispatch",
payload["contracts_by_key"]["ops_job_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_agent_protocol.md",
payload["contracts_by_key"]["ops_agent_protocol"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/agent/bootstrap-plan",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertEqual(
"/api/v1/ops/releases/launchpad",
payload["contracts_by_key"]["release_hub_contract"]["primary_endpoint"],
)
self.assertIn(
"/api/v1/ops/codex-brief",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/preview",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/resolve",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/execute-resolved",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/codex-actions/resolve",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/codex-actions/execute",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_playbook_contract.md",
payload["contracts_by_key"]["ops_playbook_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/playbooks/execute",
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/playbook-runs/{run_code}/events",
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_observability_contract.md",
payload["contracts_by_key"]["ops_observability_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/inspection-overview",
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/scene-log",
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_stack_diagnosis_contract.md",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/stack-diagnosis",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_driver_contract",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_doctor_decision_contract.md",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/doctor-decision",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_stack_diagnosis_contract",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_signoff_contract.md",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-signoff",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_stack_diagnosis_contract",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_bundle_contract.md",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-bundle",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_go_live_signoff_contract",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_review_contract.md",
payload["contracts_by_key"]["ops_go_live_review_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-review",
payload["contracts_by_key"]["ops_go_live_review_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_go_live_bundle_contract",
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
)
self.assertIn(
"ops_doctor_decision_contract",
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
)
def test_contract_detail_exposes_related_contracts(self) -> None:
payload = get_ops_contract_detail("ops_stack_diagnosis_contract")
self.assertEqual("2026-04-18", payload["registry_version"])
self.assertEqual("/api/v1/ops/contracts/ops_stack_diagnosis_contract", payload["detail_endpoint"])
self.assertEqual("ops_stack_diagnosis_contract", payload["contract"]["key"])
self.assertIn("ops_driver_contract", payload["related_contract_keys"])
self.assertIn("release_hub_contract", payload["related_contract_keys"])
self.assertEqual(
"docs/schemas/ops_driver_contract.md",
next(
item["schema_doc_path"]
for item in payload["related_contracts"]
if item["key"] == "ops_driver_contract"
),
)
self.assertEqual({}, get_ops_contract_detail("missing-contract"))
def test_capabilities_blueprint_and_runbook_expose_contract_registry(self) -> None:
capabilities = get_ops_capabilities()
contract_registry = capabilities["contract_registry"]
self.assertEqual("/api/v1/ops/contracts", contract_registry["endpoint"])
self.assertEqual("/api/v1/ops/contracts/{contract_key}", contract_registry["detail_endpoint_pattern"])
self.assertEqual(11, contract_registry["contracts_total"])
self.assertIn("ops_job_contract", contract_registry["contract_keys"])
self.assertIn("ops_driver_contract", contract_registry["contract_keys"])
self.assertIn("ops_playbook_contract", contract_registry["contract_keys"])
self.assertIn("ops_observability_contract", contract_registry["contract_keys"])
self.assertIn("ops_stack_diagnosis_contract", contract_registry["contract_keys"])
self.assertIn("ops_doctor_decision_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_signoff_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_bundle_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_review_contract", contract_registry["contract_keys"])
remote_log_capability = next(
item for item in capabilities["capabilities"] if item["key"] == "remote_log_sync"
)
self.assertEqual("ready", remote_log_capability["current_state"])
self.assertEqual(
"docs/schemas/release_hub_contract.md",
contract_registry["schema_docs"]["release_hub_contract"],
)
self.assertEqual(
"docs/schemas/ops_playbook_contract.md",
contract_registry["schema_docs"]["ops_playbook_contract"],
)
self.assertEqual(
"docs/schemas/ops_job_contract.md",
contract_registry["schema_docs"]["ops_job_contract"],
)
self.assertEqual(
"docs/schemas/ops_observability_contract.md",
contract_registry["schema_docs"]["ops_observability_contract"],
)
self.assertEqual(
"docs/schemas/ops_stack_diagnosis_contract.md",
contract_registry["schema_docs"]["ops_stack_diagnosis_contract"],
)
self.assertEqual(
"docs/schemas/ops_doctor_decision_contract.md",
contract_registry["schema_docs"]["ops_doctor_decision_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_signoff_contract.md",
contract_registry["schema_docs"]["ops_go_live_signoff_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_bundle_contract.md",
contract_registry["schema_docs"]["ops_go_live_bundle_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_review_contract.md",
contract_registry["schema_docs"]["ops_go_live_review_contract"],
)
blueprint = get_ops_blueprint()
self.assertEqual("2026-04-18", blueprint["contract_registry"]["registry_version"])
self.assertEqual(11, blueprint["contract_registry"]["contracts_total"])
with patch("app.services.ops_service.get_runtime_status") as mock_runtime_status, patch(
"app.services.ops_service.list_managed_nodes_with_agent_state"
) as mock_list_managed_nodes_with_agent_state, patch(
"app.services.ops_service.get_release_launchpad"
) as mock_get_release_launchpad, patch(
"app.services.ops_service._attach_ops_runbook_sequence_resolutions"
) as mock_attach_sequences:
mock_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {}
mock_attach_sequences.return_value = []
runbook = get_ops_runbook()
self.assertIn("GET /api/v1/ops/contracts", runbook["fast_checks"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,149 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_doctor_decision
class OpsServiceDoctorDecisionTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service._build_live_doctor_payload")
@patch("app.services.ops_service._find_latest_go_live_bundle_manifest")
def test_doctor_decision_falls_back_to_live_payload_when_bundle_missing(
self,
mock_find_latest_go_live_bundle_manifest,
mock_build_live_doctor_payload,
mock_get_ops_contract_registry,
) -> None:
mock_find_latest_go_live_bundle_manifest.return_value = (None, Path("/tmp/go-live-bundles"))
mock_build_live_doctor_payload.return_value = (
{
"status": "attention",
"reason_code": "live_fallback",
"headline": "当前先补 attention 项",
"detail": "需要先顺着 stack next 收口。",
"preferred_surface": "stack-summary",
"next_action_code": "review_stack_next",
"recommended_commands": ["bash primary"],
"evidence": {},
},
{
"ok": False,
"required_failures": [],
"optional_unavailable": [],
"scene_log_reports_total": 0,
"scene_log_reports_ok": 0,
"scene_log_status_counts": {},
"scene_log_reports": [],
"contract_surface_gaps": [],
},
)
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_doctor_decision_contract": {"version": "v1"},
}
}
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertEqual("主决策待复核", payload["status_label"])
self.assertEqual("live_fallback", payload["source"])
self.assertEqual("review_stack_next", payload["decision"]["next_action_code"])
self.assertIn("doctor-decision", payload["recommended_commands"]["doctor_decision"])
self.assertEqual("bash primary", payload["recommended_commands"]["primary_recovery"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
payload["recommended_commands"]["stack_check"],
)
self.assertEqual(
payload["recommended_commands"]["stack_check"],
payload["recommended_commands"]["stack_diagnosis"],
)
@patch("app.services.ops_service.get_ops_contract_registry")
def test_doctor_decision_reads_bundle_doctor_artifact_when_available(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_doctor_decision_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
doctor_artifact_path = report_dir / "07_doctor_decision.json"
doctor_artifact_path.write_text(
json.dumps(
{
"summary": {
"ok": False,
"required_failures": ["06_contracts_registry.txt"],
"optional_unavailable": ["12_activity_preview.txt"],
"scene_log_reports_total": 2,
"scene_log_reports_ok": 1,
"scene_log_status_counts": {"ok": 1, "missing": 1},
"scene_log_reports": [
{"node_code": "mainland-worker-01", "status": "ok", "ok": True},
{"node_code": "mainland-controller-01", "status": "missing", "ok": False},
],
"contract_surface_gaps": ["activity-stream"],
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补协议面",
"launchpad_recommended_recovery_summary": "先补 driver / activity 的协议导航。",
},
"decision": {
"status": "blocked",
"reason_code": "contracts_registry_unavailable",
"headline": "contract registry 当前不可用",
"detail": "先恢复 contracts registry 再继续自动驾驶。",
"preferred_surface": "contracts",
"next_action_code": "restore_contract_registry",
"recommended_commands": [
"bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100",
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
],
"evidence": {"surface_http_status": {"contracts_registry": 503}},
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "doctor_decision", "path": str(doctor_artifact_path), "ok": True},
],
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
self.assertEqual("blocked", payload["status"])
self.assertEqual("bundle_manifest", payload["source"])
self.assertEqual("contracts", payload["decision"]["preferred_surface"])
self.assertEqual(["06_contracts_registry.txt"], payload["summary"]["required_failures"])
self.assertEqual(["mainland-controller-01"], payload["summary"]["scene_log_problem_node_codes"])
self.assertIn("contracts", payload["recommended_commands"]["primary_recovery"])
self.assertIn(str(manifest_path), payload["recommended_commands"]["doctor_decision"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,274 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import preview_driver_action
class OpsServiceDriverActionPreviewTests(unittest.TestCase):
def test_preview_driver_action_builds_safe_auto_contract_preview(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "先补现场日志回传",
"summary": "优先打开关键日志回传。",
"reason": "当前还看不到现场过程日志。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_label": "开启关键回传",
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
self.assertEqual("ops_driver_contract", data["contract_key"])
self.assertEqual("/api/v1/ops/driver-feed", data["contract_primary_endpoint"])
self.assertEqual("/api/v1/ops/driver-actions/preview", data["preview_endpoint"])
self.assertEqual("driver_action", data["executor_kind"])
self.assertEqual(
"ops-driver -> driver-action execute -> playbook / ui-intent / settings",
data["execution_chain"],
)
primary = data["primary"]
self.assertEqual("开启关键回传", primary["label"])
self.assertEqual("enable_log_sync_key", primary["action_code"])
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertEqual("execution_scene", data["focus_ref"]["kind"])
self.assertEqual("execution_scene", data["primary_focus_ref"]["kind"])
self.assertEqual("execution_scene", primary["focus_ref"]["kind"])
self.assertEqual(
{
"action_code": "enable_log_sync_key",
"node_codes": [],
"action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"requested_by": "web-ui/ops-center",
},
primary["request_payload"],
)
self.assertFalse(data["secondary"])
def test_preview_driver_action_marks_missing_required_fields_as_blocked(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "driver-feed",
"title": "打开全量现场取证",
"summary": "当前需要进入取证工作区。",
"reason": "需要明确 playbook 上下文。",
"executor_kind": "driver_action",
"primary_label": "打开取证",
"primary_action_code": "open_playbook_dialog",
"primary_action_payload": {},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
primary = data["primary"]
self.assertEqual("当前阻断", primary["recommendation_label"])
self.assertTrue(primary["execution_profile"]["blocked"])
self.assertEqual(["playbook_key"], primary["execution_profile"]["missing_fields"])
def test_preview_driver_action_marks_release_launchpad_review_as_safe_auto(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "查看 Worker 预案",
"summary": "当前建议先复核 Worker launchpad 预案。",
"reason": "需要先判断目标节点、门禁和阻断缺口。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 88,
"section": "release_launchpad",
"mode": "worker",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 88,
"section": "release_launchpad",
"mode": "worker",
},
"primary_label": "查看 Worker 预案",
"primary_action_code": "review_smart_rollout_preview",
"primary_action_payload": {
"channel": "stable",
},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
primary = data["primary"]
self.assertEqual("review_smart_rollout_preview", primary["action_code"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertFalse(primary["execution_profile"]["ui_only"])
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
def test_preview_driver_action_marks_handover_gap_as_safe_auto(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "先补接管缺口",
"summary": "当前存在待接管节点。",
"reason": "需要先查看首台缺口节点的接管详情。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "managed_node",
"node_code": "mainland-worker-01",
},
"primary_focus_ref": {
"kind": "managed_node",
"node_code": "mainland-worker-01",
},
"primary_label": "查看接管详情",
"primary_action_code": "handover_first_gap",
"primary_action_payload": {
"node_code": "mainland-worker-01",
},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
primary = data["primary"]
self.assertEqual("handover_first_gap", primary["action_code"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertFalse(primary["execution_profile"]["ui_only"])
@patch("app.services.ops_service.get_ops_runbook")
def test_preview_driver_action_supports_runbook_sequence_primary_and_secondary(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"target_scope_label": "Release / Rollout / 2026.04.18",
"step_titles": ["确认发布包", "发灰度"],
"target_node_codes": ["mainland-controller-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "Worker 灰度",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_label": "查看版本区",
"secondary_action_code": "focus_release_hub",
"secondary_action_payload": {"release_id": 7},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
}
]
}
ok, message, data = preview_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_label": "Worker 灰度",
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
self.assertEqual("runbook_sequence", data["executor_kind"])
self.assertEqual("runbook sequence", data["executor_kind_label"])
self.assertEqual(
"ops-driver -> runbook-sequence -> resolved driver action / ui-intent",
data["execution_chain"],
)
self.assertEqual("create_release_rollout_worker", data["resolved_primary"]["driver_action_code"])
self.assertEqual("release_launchpad", data["focus_ref"]["section"])
self.assertEqual("worker", data["primary_focus_ref"]["mode"])
self.assertEqual("release_detail", data["secondary_focus_ref"]["section"])
primary = data["primary"]
self.assertEqual("/api/v1/ops/runbook/sequences/release_progression/execute", primary["target_api"])
self.assertEqual("建议确认后执行", primary["recommendation_label"])
self.assertEqual("guarded_auto", primary["automation_level_label"])
self.assertEqual("worker", primary["focus_ref"]["mode"])
self.assertEqual(
{
"secondary": False,
"requested_by": "web-ui/ops-center",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
primary["request_payload"],
)
secondary = data["secondary"]
self.assertEqual("查看版本区", secondary["label"])
self.assertEqual("focus_release_hub", secondary["action_code"])
self.assertEqual("release_detail", secondary["focus_ref"]["section"])
self.assertEqual(
{
"secondary": True,
"requested_by": "web-ui/ops-center",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
secondary["request_payload"],
)
self.assertEqual("可自动执行", secondary["recommendation_label"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,433 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_resolved_driver_action, resolve_driver_action_request
def _build_preview_data(
*,
executor_kind: str,
primary_action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
primary_request_payload: dict,
sequence_key: str = "",
secondary_action_code: str = "",
secondary_request_payload: dict | None = None,
secondary_recommendation: str = "",
secondary_automation_level: str = "",
secondary_target_api: str = "",
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
return {
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"execution_chain": "ops-driver -> preview",
"focus_ref": dict(focus_ref or {}),
"primary_focus_ref": dict(primary_focus_ref or {}),
"secondary_focus_ref": dict(secondary_focus_ref or {}),
"primary": {
"label": "主动作",
"action_code": primary_action_code,
"node_codes": ["mainland-controller-01"],
"target_api": target_api,
"target_method": "POST",
"request_payload": dict(primary_request_payload or {}),
"focus_ref": dict(primary_focus_ref or {}),
"execution_profile": {
"action_code": primary_action_code,
"backend_handled": True,
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if recommendation == "auto_execute" else "high",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
},
"secondary": (
{
"label": "次动作",
"action_code": secondary_action_code,
"node_codes": ["mainland-controller-01"],
"target_api": secondary_target_api or target_api,
"target_method": "POST",
"request_payload": dict(secondary_request_payload or {}),
"focus_ref": dict(secondary_focus_ref or {}),
"execution_profile": {
"action_code": secondary_action_code,
"backend_handled": True,
"automation_level": secondary_automation_level or automation_level,
"recommendation": secondary_recommendation or recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if (secondary_recommendation or recommendation) == "auto_execute" else "high",
"confirm_required": (secondary_recommendation or recommendation) == "confirm_then_execute",
"ui_only": (secondary_recommendation or recommendation) == "open_ui",
"blocked": (secondary_recommendation or recommendation) == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
}
if secondary_action_code
else {}
),
}
class OpsServiceDriverActionResolveTests(unittest.TestCase):
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_returns_gate_for_safe_auto(self, mock_preview_driver_action) -> None:
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/driver-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
primary_request_payload=request_payload,
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
),
)
ok, message, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "开启关键日志回传",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("驾驶动作解析完成", message)
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_requires_confirmation_for_guarded_runbook(self, mock_preview_driver_action) -> None:
request_payload = {
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload=request_payload,
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
),
)
ok, _, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("confirmation_required", data["gate"]["decision"])
self.assertFalse(data["gate"]["will_execute"])
self.assertTrue(data["gate"]["confirm_required"])
self.assertEqual("release_progression", data["gate"]["sequence_key"])
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
self.assertEqual("worker", data["focus_ref"]["mode"])
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_can_select_secondary_open_ui_section(self, mock_preview_driver_action) -> None:
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload={
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
secondary_action_code="focus_release_hub",
secondary_request_payload={
"secondary": True,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
secondary_recommendation="auto_execute",
secondary_automation_level="safe_auto",
secondary_target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
),
)
ok, _, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"primary_action_code": "create_release_rollout_worker",
"secondary_action_code": "focus_release_hub",
"primary_action_payload": {"release_id": 7},
"secondary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
"secondary": True,
}
)
self.assertTrue(ok)
self.assertEqual("secondary", data["selected_section"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
self.assertEqual("release_detail", data["focus_ref"]["section"])
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_blocks_guarded_action_without_confirm(self, mock_preview_driver_action) -> None:
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload={
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
),
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
}
)
self.assertFalse(ok)
self.assertIn("需要显式确认", message)
self.assertFalse(data["executed"])
self.assertEqual("confirmation_required", data["gate"]["decision"])
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_runs_safe_auto_driver_action(
self,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/driver-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
primary_request_payload=request_payload,
),
)
mock_execute_driver_action.return_value = (
True,
"日志回传已开启",
{"handled": True, "mode": "ops-job"},
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "开启关键日志回传",
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("日志回传已开启", message)
self.assertTrue(data["executed"])
self.assertEqual({"handled": True, "mode": "ops-job"}, data["execution_result"])
mock_execute_driver_action.assert_called_once_with(request_payload)
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_runs_guarded_runbook_after_confirm(
self,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
request_payload = {
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload=request_payload,
sequence_key="release_progression",
),
)
mock_execute_ops_runbook_sequence.return_value = (
True,
"Release rollout 已创建",
{
"handled": True,
"mode": "smart-release-rollout",
"driver_action_code": "create_release_rollout_worker",
},
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
"confirm": True,
}
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["executed"])
self.assertEqual("execute_confirmed", data["gate"]["decision"])
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,358 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_driver_feed
class OpsServiceDriverFeedTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_go_live_summary")
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.get_ops_overview")
def test_driver_feed_merges_priority_recommendation_driver_cards_and_runbook_sequences(
self,
mock_get_ops_overview,
mock_get_ops_runbook,
mock_get_ops_go_live_summary,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "blocked",
"publish_ready": False,
"publish_status": "blocked",
"publish_status_label": "暂不可发布",
"publish_summary": "当前仍有硬阻断项,不能直接进入正式发版。",
"operator_title": "先补接管缺口",
"next_step_action_code": "handover_first_gap",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 1,
"blocking_reasons": ["remote_agent_ready=0"],
"warnings": ["log_sync_waiting_sample=0/2"],
}
mock_get_ops_overview.return_value = {
"execution_scene": {
"participation_summary": {
"participating_nodes": 2,
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
},
"participating_nodes": [
{"node_code": "mainland-worker-01"},
{"node_code": "mainland-controller-01"},
],
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"participating_node_count": 2,
"covered_participating_node_count": 0,
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
},
},
"recommendation": {
"key": "fix-handover-gap",
"priority": "先补接管缺口",
"title": "先补接管缺口",
"summary": "当前应该先把接管链路补齐。",
"reason": "否则后续批量动作无法稳定下发。",
"level_label": "最高优先",
"tag_type": "danger",
"primary_label": "纳管首台节点",
"secondary_label": "查看目标节点",
"primary_action_code": "handover_first_gap",
"secondary_action_code": "view_first_gap",
"node_codes": ["mainland-worker-02"],
"primary_node_codes": ["mainland-worker-02"],
"secondary_node_codes": ["mainland-worker-02"],
"primary_action_payload": {"requested_by": "test"},
"secondary_action_payload": {},
"reasons": ["当前存在接管缺口。"],
},
"activity_stream": {
"items": [
{
"kind": "ops_job",
"activity_key": "ops-job:501",
"title": "logs.collect",
"subtitle": "ops-logs-501",
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"summary": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"status": "running",
"status_label": "执行中",
"meta_text": "ops-logs-501 / 执行 远端 Agent",
"occurred_at": "2026-04-18 09:18:00",
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "ops_job",
"job_id": 501,
"job_code": "ops-logs-501",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 501,
"event_key": "job-event:501",
},
"ui_intent": {
"kind": "job_events",
"job_id": 501,
"job_code": "ops-logs-501",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
}
]
},
"driver_recommendations": [
{
"key": "fix-handover-gap",
"title": "先补接管缺口",
"summary": "重复建议,不应再次进入 feed。",
"reason": "duplicate",
"level_label": "最高优先",
"tag_type": "danger",
"primary_label": "纳管首台节点",
"primary_action_code": "handover_first_gap",
},
{
"key": "inspect-scene-log-gap",
"title": "排查现场日志样本缺口",
"summary": "现场日志样本还没有形成。",
"reason": "先看 Worker 日志。",
"meta_text": "参与节点 2 台 / 已回传来源 0 台",
"level_label": "推荐",
"tag_type": "warning",
"primary_label": "看 Worker 日志",
"secondary_label": "执行标准巡检",
"primary_action_code": "open_worker_logs",
"secondary_action_code": "run_standard_inspection",
"node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_action_payload": {"lines": 120},
"secondary_action_payload": {},
},
],
}
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"summary": "当前可进入发布路径。",
"reason": "正式运维时应统一落回 Release / Rollout。",
"status": "warning",
"status_label": "待处理",
"tag_type": "warning",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "补执行面接管",
"secondary_label": "查看版本区",
"primary_action_code": "fix_managed_nodes",
"secondary_action_code": "focus_release_hub",
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "default_rollout_gate",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"step_titles": ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入"],
"primary_resolution": {
"ok": True,
"driver_action_code": "fix_managed_nodes",
"driver_action_label": "补执行面接管",
"driver_node_codes": ["mainland-controller-01"],
"driver_action_payload": {"channel": "stable"},
"resolved_at": "2026-04-18T03:00:00",
},
"secondary_resolution": {
"ok": True,
"driver_action_code": "focus_release_hub",
"driver_action_label": "查看版本区",
"driver_node_codes": [],
"driver_action_payload": {"release_id": 7},
},
}
]
}
payload = get_ops_driver_feed()
self.assertEqual(3, payload["summary"]["total"])
self.assertEqual("blocked", payload["go_live_summary"]["go_live_status"])
self.assertEqual("blocked", payload["summary"]["go_live_status"])
self.assertEqual("先补接管缺口", payload["summary"]["operator_title"])
self.assertEqual("blocked", payload["summary"]["publish_status"])
self.assertFalse(payload["summary"]["publish_ready"])
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
self.assertIn("发布闸门:暂不可发布", payload["headline"])
self.assertIn("接入缺口签发接入工单mainland-controller-01", payload["headline"])
self.assertEqual("handover_first_gap", payload["go_live_summary"]["next_step_action_code"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["summary"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(2, payload["summary"]["do_now_total"])
self.assertEqual(1, payload["summary"]["standard_path_total"])
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_playbook_contract", payload["contract_navigation"]["contract_keys"])
self.assertIn("ops_observability_contract", payload["contract_navigation"]["contract_keys"])
self.assertEqual("priority:fix-handover-gap", payload["top_recommendation"]["key"])
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
self.assertEqual(2, len(payload["driver_recommendations"]))
self.assertEqual(1, len(payload["runbook_sequences"]))
self.assertEqual(1, len(payload["activity_focus"]))
self.assertEqual(
["priority_recommendation", "driver_recommendation", "runbook_sequence"],
[item["kind"] for item in payload["entries"]],
)
self.assertEqual(1, payload["summary"]["activity_focus_total"])
self.assertEqual("disabled", payload["summary"]["scene_log_status"])
self.assertEqual("blocked", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual(2, payload["summary"]["preview_only_total"])
self.assertEqual(3, payload["summary"]["backend_handled_total"])
self.assertEqual("blocked", payload["automation_coverage"]["launch_status"])
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(0, payload["automation_coverage"]["guarded_auto_total"])
self.assertEqual(2, payload["automation_coverage"]["auto_execute_total"])
self.assertEqual(1, payload["automation_coverage"]["resolve_first_total"])
self.assertEqual(2, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
self.assertEqual(1, payload["automation_coverage"]["human_dependency_total"])
self.assertFalse(payload["automation_coverage"]["launch_ready"])
priority_entry = payload["entries"][0]
self.assertEqual("do_now", priority_entry["lane"])
self.assertEqual("driver_action", priority_entry["executor_kind"])
self.assertEqual("handover_first_gap", priority_entry["primary_action_code"])
self.assertEqual("ops_driver_contract", priority_entry["contract_navigation"]["primary_contract_key"])
activity_focus = payload["activity_focus"][0]
self.assertEqual("ops_job", activity_focus["activity_kind"])
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", activity_focus["summary"])
self.assertEqual("focus_latest_job_events", activity_focus["focus_action_code"])
self.assertEqual("job_events", activity_focus["ui_intent_kind"])
self.assertEqual("ops_job_event", activity_focus["source_focus_ref"]["kind"])
self.assertTrue(activity_focus["observation_only"])
self.assertEqual("ops_observability_contract", activity_focus["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", activity_focus["contract_navigation"]["contract_keys"])
scene_log_observation = payload["scene_log_observation"]
self.assertEqual("disabled", scene_log_observation["status"])
self.assertEqual("mainland-worker-01", scene_log_observation["target_node_code"])
self.assertEqual("node_scene_log", scene_log_observation["focus_ref"]["kind"])
self.assertEqual(
"ops_observability_contract",
scene_log_observation["contract_navigation"]["primary_contract_key"],
)
runbook_entry = payload["entries"][2]
self.assertEqual("runbook_sequence", runbook_entry["kind"])
self.assertEqual("standard_path", runbook_entry["lane"])
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
self.assertEqual("release_progression", runbook_entry["sequence_key"])
self.assertEqual("fix_managed_nodes", runbook_entry["primary_action_code"])
self.assertEqual("focus_release_hub", runbook_entry["secondary_action_code"])
self.assertIn("当前解析:补执行面接管", runbook_entry["detail_lines"][0])
self.assertEqual("release_hub", runbook_entry["focus_ref"]["kind"])
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
self.assertEqual("default_rollout_gate", runbook_entry["primary_focus_ref"]["section"])
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
self.assertEqual("default_rollout_gate", runbook_entry["primary_action_payload"]["focus_ref"]["section"])
self.assertEqual("release_detail", runbook_entry["secondary_action_payload"]["focus_ref"]["section"])
self.assertEqual("ops_driver_contract", runbook_entry["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_playbook_contract", runbook_entry["contract_navigation"]["contract_keys"])
self.assertIn("release_hub_contract", runbook_entry["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_go_live_summary")
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.get_ops_overview")
def test_driver_feed_preserves_bootstrap_run_as_top_recommendation(
self,
mock_get_ops_overview,
mock_get_ops_runbook,
mock_get_ops_go_live_summary,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
"publish_ready": False,
"publish_status": "attention",
"publish_status_label": "可发布但建议先复核",
"publish_summary": "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。",
"operator_title": "先跑接入收口",
"next_step_action_code": "bootstrap_run",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "跑接入收口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"blocking_reasons": [],
"warnings": ["remote_agent_ready=1/2"],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"key": "handover-gap-bootstrap_run",
"priority": "先补接管缺口",
"title": "先补接管缺口",
"summary": "Launchpad 已明确收敛到首台节点接入收口。",
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"level_label": "最高优先",
"tag_type": "warning",
"primary_label": "跑接入收口",
"secondary_label": "查看目标节点",
"primary_action_code": "bootstrap_run",
"secondary_action_code": "view_first_gap",
"node_codes": ["mainland-worker-01"],
"primary_node_codes": ["mainland-worker-01"],
"secondary_node_codes": ["mainland-worker-01"],
"primary_action_payload": {"node_code": "mainland-worker-01"},
"secondary_action_payload": {},
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"driver_recommendations": [],
"activity_stream": {"items": []},
"execution_scene": {},
}
mock_get_ops_runbook.return_value = {"control_sequences": []}
payload = get_ops_driver_feed()
self.assertEqual("priority:handover-gap-bootstrap_run", payload["top_recommendation"]["key"])
self.assertEqual("attention", payload["summary"]["publish_status"])
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-worker-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("跑接入收口", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual("attention", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("bootstrap_run", payload["top_recommendation"]["primary_action_code"])
self.assertEqual("mainland-worker-01", payload["top_recommendation"]["primary_node_codes"][0])
self.assertIn("ops_agent_protocol", payload["top_recommendation"]["contract_navigation"]["contract_keys"])
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
self.assertEqual(0, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
self.assertEqual(0, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(1, payload["automation_coverage"]["backend_handled_total"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,24 @@
import unittest
from app.core.config import settings
from app.services.ops_service import _recommend_driver_execution_mode_for_supported_modes
class OpsServiceExecutionModeRecommendationTests(unittest.TestCase):
def test_recommend_driver_execution_mode_prefers_local_runtime_for_current_node(self) -> None:
result = _recommend_driver_execution_mode_for_supported_modes(
[settings.node_code],
supported_modes=["remote-agent", "local-runtime", "ssh"],
default_mode="remote-agent",
)
self.assertEqual("local-runtime", result["execution_mode"])
self.assertIn("local-runtime", result["reason"])
self.assertEqual(
["remote-agent", "local-runtime", "ssh"],
result["supported_modes"],
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,106 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_bundle
class OpsServiceGoLiveBundleTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_bundle_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_bundle_contract": {"version": "v1"},
}
}
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
self.assertEqual("missing", payload["status"])
self.assertFalse(payload["bundle_available"])
self.assertEqual("待导出", payload["status_label"])
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_bundle_reads_manifest_summary_and_review_state(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_bundle_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
env_audit_path = report_dir / "00_env_audit.txt"
env_audit_path.write_text(
"[8/8] condensed env audit summary\n"
+ json.dumps(
{
"status": "attention",
"headline": "仍有轻微环境缺口",
"missing_items": ["missing_tool:jq"],
"runtime": {"runtime_may_need_restart": True},
"recommended_actions": ["bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover"],
},
ensure_ascii=False,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
{"key": "go_live_summary", "path": str(report_dir / "02_go_live_summary.json"), "ok": True},
],
"failures": ["doctor_export"],
"summary": {
"ok": False,
"artifact_total": 9,
"failure_total": 1,
"review_status_hint": "attention",
"review_headline": "核心报告已齐,但建议先复核补充缺口。",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接管缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"launchpad_alignment": {"consistent": False, "target_node_code_consistent": False},
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
"contract_surface_gaps": ["ops_go_live_bundle_contract"],
"discovered_contract_keys": ["ops_go_live_signoff_contract"],
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertTrue(payload["bundle_available"])
self.assertEqual(str(manifest_path), payload["manifest_path"])
self.assertEqual(9, payload["artifact_total"])
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
self.assertEqual("attention", payload["env_audit"]["status"])
self.assertTrue(payload["env_audit"]["runtime_may_need_restart"])
self.assertFalse(payload["summary"]["launchpad_alignment"]["consistent"])
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_signoff"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,169 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_review
class OpsServiceGoLiveReviewTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_review_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_review_contract": {"version": "v1"},
}
}
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
self.assertEqual("missing", payload["status"])
self.assertEqual("待导出", payload["status_label"])
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_review_reads_manifest_and_alignment(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_review_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
env_audit_path = report_dir / "00_env_audit.txt"
env_audit_path.write_text(
"[8/8] condensed env audit summary\n"
+ json.dumps(
{
"status": "attention",
"headline": "环境仍有轻微缺口",
"missing_items": ["missing_tool:jq"],
"recommended_actions": ["bash runtime-refresh-recover"],
},
ensure_ascii=False,
),
encoding="utf-8",
)
go_live_summary_path = report_dir / "02_go_live_summary.json"
go_live_summary_path.write_text(
json.dumps(
{
"go_live_status": "attention",
"publish_status": "attention",
"operator_title": "先补接入缺口",
"next_step_action_code": "fix_managed_nodes",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
ensure_ascii=False,
),
encoding="utf-8",
)
stack_diagnosis_path = report_dir / "03_stack_diagnosis.json"
stack_diagnosis_path.write_text(
json.dumps(
{
"diagnosis": {
"stack_status": "attention",
"operator_decision": {"title": "先补接入缺口"},
"next_step": {"action_code": "fix_managed_nodes"},
"launchpad_recommended_target_node_code": "mainland-worker-02",
"launchpad_recommended_recovery_label": "先看节点接管",
"launchpad_recommended_recovery_summary": "stack 与 summary 存在漂移。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
}
},
ensure_ascii=False,
),
encoding="utf-8",
)
driver_feed_path = report_dir / "04_driver_feed.json"
driver_feed_path.write_text(
json.dumps(
{
"headline": "driver ok",
"summary": {
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
},
ensure_ascii=False,
),
encoding="utf-8",
)
codex_brief_path = report_dir / "05_codex_brief.json"
codex_brief_path.write_text(
json.dumps(
{
"headline": "codex ok",
"summary": {
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
"focus": {"entry_key": "fix_managed_nodes"},
},
ensure_ascii=False,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
{"key": "go_live_summary", "path": str(go_live_summary_path), "ok": True},
{"key": "stack_diagnosis", "path": str(stack_diagnosis_path), "ok": True},
{"key": "driver_feed", "path": str(driver_feed_path), "ok": True},
{"key": "codex_brief", "path": str(codex_brief_path), "ok": True},
{"key": "release_launchpad", "path": str(report_dir / "06_release_launchpad.json"), "ok": True},
],
"failures": ["doctor_export"],
"summary": {
"artifact_total": 9,
"failure_total": 1,
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertEqual(9, payload["artifact_total"])
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
self.assertFalse(payload["launchpad_alignment"]["consistent"])
self.assertIn("go_live_summary", payload["launchpad_alignment"]["available_sources"])
self.assertIn("mainland-worker-01", payload["recommended_next_steps"][0] + "".join(payload["recommended_next_steps"]))
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_review"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,317 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_signoff
class OpsServiceGoLiveSignoffTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_ready_when_all_surfaces_align(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "ready",
"publish_status": "ready",
"operator_title": "进入发布",
"next_step_action_code": "publish_latest_worker",
"next_step_reason": "当前门禁已收口。",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {"next_step": "bash next-step"},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"stack_status": "ready",
"operator_decision": {"title": "进入发布"},
"next_step": {"action_code": "publish_latest_worker"},
}
}
mock_get_ops_driver_feed.return_value = {
"headline": "主车道清晰",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "可签字",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {"entry_key": "publish_latest_worker"},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "继续发布",
"recommended_recovery_summary": "可以进入正式发布。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "ready",
"status_label": "可复核发布",
"headline": "review ok",
"report_dir": "/tmp/review",
"manifest_path": "/tmp/review/manifest.json",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "ready",
"headline": "doctor ok",
"report_dir": "/tmp/review",
"manifest_path": "/tmp/review/manifest.json",
"decision": {
"reason_code": "all_green",
"preferred_surface": "go-live-summary",
"next_action_code": "publish_latest_worker",
},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff(base_url="http://127.0.0.1:8100")
self.assertEqual("ready", payload["signoff_status"])
self.assertEqual("可签字上线", payload["status_label"])
self.assertEqual([], payload["blocked_reasons"])
self.assertEqual([], payload["attention_reasons"])
self.assertTrue(payload["launchpad_alignment"]["consistent"])
self.assertEqual("go-live-summary", payload["decision"]["preferred_surface"])
self.assertIn("go-live-signoff", payload["recommended_commands"]["go_live_signoff"])
self.assertEqual("ready", payload["release_gate"]["go_live_review_status"])
self.assertEqual("ready", payload["release_gate"]["doctor_decision_status"])
self.assertEqual("/tmp/review/manifest.json", payload["manifest_path"])
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_attention_when_launchpad_alignment_drifts(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "ready",
"publish_status": "ready",
"operator_title": "进入发布",
"next_step_action_code": "publish_latest_worker",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {"stack_status": "ready", "operator_decision": {}, "next_step": {}}
}
mock_get_ops_driver_feed.return_value = {
"headline": "主车道清晰",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-02",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "可签字",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "继续发布",
"recommended_recovery_summary": "可以进入正式发布。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "attention",
"headline": "review drift",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "ready",
"headline": "doctor ok",
"decision": {"preferred_surface": "go-live-summary"},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff()
self.assertEqual("attention", payload["signoff_status"])
self.assertIn("launchpad_alignment=inconsistent", payload["attention_reasons"])
self.assertIn("go_live_review=attention", payload["attention_reasons"])
self.assertFalse(payload["launchpad_alignment"]["consistent"])
self.assertFalse(payload["launchpad_alignment"]["target_node_code_consistent"])
self.assertEqual("go-live-review", payload["decision"]["preferred_surface"])
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_blocked_when_publish_gate_is_blocked(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
"publish_status": "blocked",
"operator_title": "先处理发布阻断",
"next_step_action_code": "fix_rollout_blockers",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"stack_status": "attention",
"operator_decision": {"title": "先看发布门禁"},
"next_step": {"action_code": "fix_rollout_blockers"},
}
}
mock_get_ops_driver_feed.return_value = {
"headline": "发布门禁阻断",
"summary": {
"launch_status": "attention",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "需先排阻断",
"summary": {
"launch_status": "attention",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {"entry_key": "fix_rollout_blockers"},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "blocked",
"status_label": "发布阻断",
"recommended_action_code": "fix_rollout_blockers",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "处理发布阻断",
"recommended_recovery_summary": "当前 rollout gate 未通过。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "ready",
"headline": "review ok",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "attention",
"headline": "doctor focus",
"decision": {
"reason_code": "publish_blocked",
"preferred_surface": "release-launchpad",
"next_action_code": "fix_rollout_blockers",
},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff()
self.assertEqual("blocked", payload["signoff_status"])
self.assertIn("publish_status=blocked", payload["blocked_reasons"])
self.assertIn("release_launchpad=blocked", payload["blocked_reasons"])
self.assertEqual("release-launchpad", payload["decision"]["preferred_surface"])
self.assertEqual("publish_blocked", payload["decision"]["doctor_reason_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,664 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_summary
class OpsServiceGoLiveSummaryTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {
"action_code": "api-restart",
"reason": "build schema stale",
"focus_ref": {"kind": "runtime_build_info", "section": "schema"},
},
"operator_decision": {
"lane": "runtime_recovery",
"title": "优先重启控制面 API 并复检运行时版本",
"primary_command_key": "api-restart",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
},
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先重启 API",
"reason": "build schema stale",
"primary_action_code": "api-restart",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual("attention", payload["publish_status"])
self.assertTrue(payload["runtime_schema_stale"])
self.assertFalse(payload["route_surface_declares_bootstrap_plan"])
self.assertIn("runtime_build_schema_stale", payload["warnings"])
self.assertIn("runtime_build_schema_stale", payload["publish_warnings"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100",
payload["recommended_commands"]["go_live_recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["api_restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["runtime_refresh_recover"],
)
self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_highlights_remote_access_and_log_sync_gaps(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "handover_first_gap",
"reason": "来自 overview.recommendation.primary_action_code",
},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_hub"},
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 1,
"recommended_target_node_code": "mainland-controller-01",
"recommended_recovery_label": "签发接入工单",
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "当前仍有节点没有进入标准执行器就绪状态。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "waiting_sample",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 0,
"missing_participating_nodes": [
"mainland-worker-01",
"mainland-controller-01",
"overseas-control-01",
],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("blocked", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
self.assertEqual("暂不可发布", payload["publish_status_label"])
self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"])
self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"])
self.assertIn("release_launchpad=attention", payload["warnings"])
self.assertIn("release_launchpad=attention", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"])
self.assertTrue(payload["route_surface_complete"])
self.assertEqual("先补接管缺口", payload["operator_title"])
self.assertEqual("handover_first_gap", payload["next_step_action_code"])
self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
payload["recommended_commands"]["next_step"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating",
payload["recommended_commands"]["log_sync_logs"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "bootstrap_run",
"reason": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"operator_decision": {
"lane": "node_handover",
"title": "优先执行节点接管收口",
"primary_command_key": "bootstrap_run",
},
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {},
},
"repository_capabilities": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "overview 仍在泛化描述。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["next_step_action_code"])
self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "run_inspection_participating",
"reason": "继续做上线前复核。",
},
"operator_decision": {
"lane": "go-live",
"title": "可以进入上线复核",
"primary_command_key": "run_inspection_participating",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "继续复核",
"reason": "当前执行面与观测面已收口。",
"primary_action_code": "run_inspection_participating",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("ready", payload["go_live_status"])
self.assertTrue(payload["publish_ready"])
self.assertEqual("ready", payload["publish_status"])
self.assertEqual("可发布", payload["publish_status_label"])
self.assertEqual([], payload["blocking_reasons"])
self.assertEqual([], payload["warnings"])
self.assertEqual([], payload["publish_blocking_reasons"])
self.assertEqual([], payload["publish_warnings"])
self.assertEqual("go-live", payload["operator_lane"])
self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"])
self.assertEqual(2, payload["log_sync_covered_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 1,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先清死信",
"reason": "当前仍有节点回执死信。",
"primary_action_code": "delivery_queue_replay",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补大陆接管",
"reason": "当前只有海外样本已形成。",
"primary_action_code": "bootstrap_run",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "partial_coverage",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 1,
"covered_participating_nodes": ["overseas-control-01"],
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual(1, payload["log_sync_covered_nodes"])
self.assertIn("log_sync_partial=1/3", payload["warnings"])
self.assertIn("log_sync_partial=1/3", payload["publish_warnings"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_attention_for_start_delivery_gap(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "focus_latest_job_events",
"reason": "先看任务事件。",
},
"operator_decision": {
"lane": "ops_jobs",
"title": "优先检查任务事件流",
"primary_command_key": "focus_latest_job_events",
},
"issues": [
{
"code": "ops_job_start_delivery_failed_local",
"summary": "最近有任务开始回执异常。",
}
],
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先看任务事件",
"reason": "最近出现开始回执异常。",
"primary_action_code": "focus_latest_job_events",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 1,
"covered_participating_node_count": 1,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("attention", payload["publish_status"])
self.assertEqual(1, payload["activity_start_delivery_issue_total"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"])
self.assertEqual("ops_jobs", payload["operator_lane"])
self.assertEqual("focus_latest_job_events", payload["next_step_action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,182 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_link_snapshot
class OpsServiceLinkSnapshotTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_overview")
def test_link_snapshot_highlights_log_sync_gap_on_active_scene(self, mock_get_ops_overview) -> None:
mock_get_ops_overview.return_value = {
"current_topology": {
"node_code": "overseas-control-01",
"node_region": "overseas",
"node_role": "control",
"worker_mode": "linux-systemd",
"sync_push_enabled": True,
"sync_target_api_base_url": "http://152.53.37.118:8100",
},
"cluster_summary": {
"online_control_nodes": 2,
"online_worker_nodes": 3,
"dedicated_online_worker_nodes": 1,
"busy_nodes": ["mainland-worker-01", "overseas-control-01"],
},
"runtime_summary": {
"readiness_status": "ready",
"readiness_summary": "当前多机与跨地域骨架已进入可联调状态。",
"phase_label": "running",
"phase_detail": "当前任务正在推进",
"active_job_code": "detect-20260417164852-75f7cc",
"progress_percent": 42.5,
},
"execution_scene": {
"participation_summary": {
"effective_online_nodes": 3,
"participating_nodes": 3,
"dispatch_active_nodes": 2,
"recent_only_nodes": 1,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01", "overseas-control-01"],
"non_participating_node_codes": [],
},
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 3,
"missing_participating_nodes": [
"mainland-controller-01",
"mainland-worker-01",
"overseas-control-01",
],
"last_at": "",
"preview_lines": [],
},
},
"sync_summary": {
"enabled": True,
"source_region": "mainland",
"target_region": "overseas",
"projected_batches": 0,
"failed_batches": 0,
"records_total": 2513,
},
"recommendation": {
"key": "enable-scene-log-sync",
"priority": "先补现场日志回传",
"summary": "当前有 3 台节点正在真实参与检测,但海外控制面还看不到过程日志。",
"reason": "先把远端日志回传切到关键模式,才能看到阶段、异常、代理与执行过程。",
"primary_label": "开启关键回传",
"primary_action_code": "enable_log_sync_key",
"primary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
"primary_action_payload": {},
"secondary_label": "开启全量回传",
"secondary_action_code": "enable_log_sync_full",
"secondary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
"secondary_action_payload": {},
},
}
snapshot = get_ops_link_snapshot()
self.assertEqual("attention", snapshot["status"])
self.assertIn("日志回传", snapshot["headline"])
self.assertIn("活跃任务 detect-20260417164852-75f7cc", snapshot["summary_lines"][2])
self.assertIn("远端日志:关闭", snapshot["summary_lines"][4])
self.assertEqual("enable_log_sync_key", snapshot["next_actions"][0]["action_code"])
self.assertEqual(3, snapshot["execution"]["participating_nodes"])
self.assertFalse(snapshot["log_sync"]["enabled"])
@patch("app.services.ops_service.get_ops_overview")
def test_link_snapshot_reports_ready_when_scene_is_observable(self, mock_get_ops_overview) -> None:
mock_get_ops_overview.return_value = {
"current_topology": {
"node_code": "overseas-control-01",
"node_region": "overseas",
"node_role": "control",
"worker_mode": "linux-systemd",
"sync_push_enabled": True,
"sync_target_api_base_url": "http://152.53.37.118:8100",
},
"cluster_summary": {
"online_control_nodes": 2,
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"busy_nodes": ["mainland-worker-01"],
},
"runtime_summary": {
"readiness_status": "ready",
"readiness_summary": "ok",
"phase_label": "running",
"phase_detail": "检测任务正常推进",
"active_job_code": "detect-001",
"progress_percent": 66.0,
},
"execution_scene": {
"participation_summary": {
"effective_online_nodes": 2,
"participating_nodes": 2,
"dispatch_active_nodes": 2,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"non_participating_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 18,
"source_node_count": 2,
"covered_participating_node_count": 2,
"participating_node_count": 2,
"missing_participating_nodes": [],
"last_at": "2026-04-17 22:20:00",
"preview_lines": [
"[2026-04-17 22:19:59] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-17 22:20:00] [mainland-controller-01] 域名检测完成: beta.com",
],
},
},
"sync_summary": {
"enabled": True,
"source_region": "mainland",
"target_region": "overseas",
"projected_batches": 0,
"failed_batches": 0,
"records_total": 2600,
},
"recommendation": {
"key": "inspect-participating",
"priority": "先看执行现场",
"summary": "当前有 2 台节点正在真实参与检测,优先回收它们的标准巡检结果。",
"reason": "真正参与检测的节点最接近现场。",
"primary_label": "执行标准巡检",
"primary_action_code": "run_inspection_participating",
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_action_payload": {},
"secondary_label": "打开诊断模板",
"secondary_action_code": "open_diagnostics_participating",
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"secondary_action_payload": {},
},
}
snapshot = get_ops_link_snapshot()
self.assertEqual("ready", snapshot["status"])
self.assertIn("可观测状态", snapshot["headline"])
self.assertIn("覆盖 2/2", snapshot["summary_lines"][4])
self.assertEqual(2, snapshot["log_sync"]["source_node_count"])
self.assertEqual(2, len(snapshot["next_actions"]))
self.assertEqual("run_inspection_participating", snapshot["next_actions"][0]["action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,256 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_node_scene_log
class OpsServiceNodeSceneLogTests(unittest.TestCase):
def _runtime_payload_with_worker_logs(self) -> dict:
return {
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 2,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"source_node_summaries": [
{
"node_code": "mainland-worker-01",
"line_count": 2,
"key_line_count": 2,
"full_line_count": 0,
"last_at": "2026-04-18 05:02:00",
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
}
],
"preview_lines": [
"[2026-04-18 05:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
],
"last_at": "2026-04-18 05:02:00",
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 1,
"standby_nodes": 1,
"load_syncing_nodes": 0,
"effective_online_nodes": 2,
"dedicated_worker_nodes": 2,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
"non_participating_node_codes": ["mainland-worker-02"],
"standby_node_codes": ["mainland-worker-02"],
"load_syncing_node_codes": [],
"summary": "有效执行节点 2 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 1 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 2,
"last_heartbeat_at": "2026-04-18 05:02:10",
"participation_state": "claimed",
"participation_label": "已领待跑",
"participation_reason": "已领取 2 项任务,等待线程继续执行。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"last_heartbeat_at": "2026-04-18 05:02:11",
"participation_state": "standby",
"participation_label": "在线待命",
"participation_reason": "当前节点在线,但尚未参与本轮检测。",
"is_dispatch_active": False,
}
],
"active_job": {
"job_code": "detect-001",
"status": "running",
"current_cycle_token": "cycle-1",
"current_cycle_events": [
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:02:00",
"node_code": "mainland-worker-01",
"message": "域名检测完成: alpha.com",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:01:00",
"node_code": "mainland-worker-01",
"message": "代理池刷新完成",
"payload": {"cycle_token": "cycle-1", "log_mode": "full"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:00:00",
"node_code": "mainland-worker-01",
"message": "开始检测域名: alpha.com",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 04:59:00",
"node_code": "mainland-worker-01",
"message": "旧周期日志",
"payload": {"cycle_token": "cycle-0", "log_mode": "key"},
},
],
},
},
"cluster": {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 2,
"last_heartbeat_at": "2026-04-18 05:02:10",
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"last_heartbeat_at": "2026-04-18 05:02:11",
},
]
},
}
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_filters_key_mode_records_for_participating_node(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
payload = get_ops_node_scene_log("mainland-worker-01", limit=10)
self.assertEqual("mainland-worker-01", payload["node_code"])
self.assertEqual("healthy", payload["status"])
self.assertEqual("关键覆盖", payload["status_label"])
self.assertEqual("key", payload["mode"])
self.assertTrue(payload["log_sync_enabled"])
self.assertTrue(payload["participation"]["detect_participating"])
self.assertEqual(2, payload["records_total"])
self.assertEqual(
[
"开始检测域名: alpha.com",
"域名检测完成: alpha.com",
],
[item["message"] for item in payload["records"]],
)
self.assertEqual(2, payload["source_summary"]["line_count"])
self.assertEqual(2, payload["source_summary"]["key_line_count"])
self.assertEqual(0, payload["source_summary"]["full_line_count"])
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_can_request_full_mode_records(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
payload = get_ops_node_scene_log("mainland-worker-01", limit=10, mode="full")
self.assertEqual("full_capture", payload["status"])
self.assertEqual("full", payload["mode"])
self.assertEqual(3, payload["records_total"])
self.assertEqual(
[
"开始检测域名: alpha.com",
"代理池刷新完成",
"域名检测完成: alpha.com",
],
[item["message"] for item in payload["records"]],
)
self.assertEqual(3, payload["source_summary"]["line_count"])
self.assertEqual(2, payload["source_summary"]["key_line_count"])
self.assertEqual(1, payload["source_summary"]["full_line_count"])
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_marks_waiting_sample_for_missing_participating_node(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = {
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"source_node_summaries": [],
"preview_lines": [],
"last_at": "",
"last_line": "",
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"effective_online_nodes": 1,
"dedicated_worker_nodes": 1,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-02"],
"recent_only_node_codes": [],
"non_participating_node_codes": [],
"standby_node_codes": [],
"load_syncing_node_codes": [],
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"last_heartbeat_at": "2026-04-18 05:05:00",
"participation_state": "running",
"participation_label": "正在执行",
"participation_reason": "当前存在执行中工作负载。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
"active_job": {"job_code": "detect-002", "status": "running", "current_cycle_events": []},
},
"cluster": {
"nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"last_heartbeat_at": "2026-04-18 05:05:00",
}
]
},
}
payload = get_ops_node_scene_log("mainland-worker-02")
self.assertEqual("waiting_sample", payload["status"])
self.assertEqual("等待样本", payload["status_label"])
self.assertEqual(0, payload["records_total"])
self.assertTrue(payload["participation"]["detect_participating"])
self.assertIn("现场样本尚未形成", payload["summary"])
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,283 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_runbook
class OpsServiceRunbookTests(unittest.TestCase):
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_includes_control_sequences(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "ready",
"summary": "当前多机现场已进入标准作业状态",
},
"worker": {
"running": True,
},
"sync_agent": {
"running": True,
},
"detect": {
"participation_summary": {
"summary": "worker 正在执行",
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
},
{
"node_code": "mainland-worker-01",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "busy",
"cluster_is_effective_worker": True,
"is_current_participant": True,
},
]
}
mock_get_latest_release.side_effect = lambda channel="stable": (
{
"id": 15,
"release_version": "2026.04.17",
"channel": "stable",
}
if channel == "stable"
else {}
)
mock_get_release_launchpad.return_value = {
"channel": "stable",
"latest_release": {
"id": 15,
"release_version": "2026.04.17",
"channel": "stable",
},
"launchpad_status": {
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。",
"recommended_action_code": "publish_latest_worker",
"focus_ref": {
"kind": "release_hub",
"release_id": 15,
"section": "release_launchpad",
},
},
"worker_rollout_preview": {
"message": "智能 Rollout 预检通过",
"smart_rollout": {
"available": True,
"summary": "已按 Worker 模式预选 1 台节点。",
"target_node_codes": ["mainland-worker-01"],
},
},
"control_rollout_preview": {
"message": "",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
},
}
runbook = get_ops_runbook()
mock_list_managed_nodes_with_agent_state.assert_called_once_with(
participation_payload=mock_get_runtime_status.return_value["detect"]
)
self.assertIn("GET /api/v1/ops/releases/launchpad", runbook["fast_checks"])
self.assertEqual("ready", runbook["release_launchpad"]["launchpad_status"]["status"])
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertIn("node_onboarding", sequences)
self.assertIn("scene_observe", sequences)
self.assertIn("standard_inspection", sequences)
self.assertIn("release_progression", sequences)
self.assertEqual("bootstrap_run", sequences["node_onboarding"]["primary_action_code"])
self.assertEqual("run_acceptance", sequences["node_onboarding"]["secondary_action_code"])
self.assertEqual("run_standard_inspection", sequences["standard_inspection"]["primary_action_code"])
self.assertEqual("open_diagnostics", sequences["standard_inspection"]["secondary_action_code"])
self.assertEqual(["mainland-worker-01"], sequences["scene_observe"]["target_node_codes"])
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_action_code"])
self.assertEqual(15, sequences["release_progression"]["primary_action_payload"]["release_id"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["primary_resolution"]["driver_node_codes"])
self.assertEqual("release_hub", sequences["release_progression"]["focus_ref"]["kind"])
self.assertEqual("release_launchpad", sequences["release_progression"]["focus_ref"]["section"])
self.assertEqual("worker", sequences["release_progression"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["mode"])
self.assertEqual("release_detail", sequences["release_progression"]["secondary_focus_ref"]["section"])
self.assertTrue(sequences["scene_observe"]["secondary_resolution"]["ok"])
self.assertEqual("run_scene_logs_full", sequences["scene_observe"]["secondary_resolution"]["driver_action_code"])
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_release_progression_can_fall_back_to_bootstrap_run(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "attention",
"summary": "当前还需补执行面接管",
},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {"participation_summary": {"summary": ""}},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"role": "worker",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
}
]
}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {
"channel": "stable",
"latest_release": {},
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"recommended_action_code": "bootstrap_run",
"focus_ref": {
"kind": "release_hub",
"release_id": 19,
"section": "release_launchpad",
},
},
"worker_rollout_preview": {
"message": "当前没有满足角色条件的在线节点可用于智能 Rollout。",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
"policy_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-02",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"remote_agent_ready": False,
"inspection_status": "missing",
"recovery_action": "bootstrap_run",
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"agent_reason": "节点尚未纳管,当前不能稳定下发 remote-agent 动作。",
}
]
}
}
},
},
"control_rollout_preview": {
"message": "",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
},
}
runbook = get_ops_runbook()
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_action_code"])
self.assertEqual(["mainland-worker-02"], sequences["release_progression"]["target_node_codes"])
self.assertEqual("待补执行面", sequences["release_progression"]["status_label"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_focus_ref"]["section"])
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["section"])
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_release_progression_falls_back_to_direct_worker_rollout_when_release_exists(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "ready",
"summary": "当前多机现场已进入标准作业状态",
},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {"participation_summary": {"summary": ""}},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
}
]
}
mock_get_latest_release.return_value = {
"id": 18,
"release_version": "2026.04.18",
"channel": "stable",
}
mock_get_release_launchpad.return_value = {}
runbook = get_ops_runbook()
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_action_code"])
self.assertEqual("focus_release_hub", sequences["release_progression"]["secondary_action_code"])
self.assertEqual(18, sequences["release_progression"]["primary_action_payload"]["release_id"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,255 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_ops_runbook_sequence, resolve_ops_runbook_sequence
class OpsServiceRunbookSequenceTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_runbook")
def test_resolve_ops_runbook_sequence_uses_primary_action(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "可以直接发布。",
"reason": "launchpad 已就绪",
"target_scope_label": "Release / Launchpad / 2026.04.18",
"step_titles": ["确认发布包", "发灰度"],
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 12,
"section": "release_launchpad",
},
"primary_label": "直接发 Worker 灰度",
"primary_action_code": "publish_latest_worker",
"primary_action_payload": {
"channel": "stable",
"release_id": 12,
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 12,
"section": "release_launchpad",
"mode": "worker",
},
}
]
}
ok, message, data = resolve_ops_runbook_sequence(
"release_progression",
{
"requested_by": "test",
},
)
self.assertTrue(ok, message)
self.assertEqual("标准作业路径解析成功", message)
self.assertEqual("publish_latest_worker", data["driver_action_code"])
self.assertEqual("直接发 Worker 灰度", data["driver_action_label"])
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
self.assertEqual("test/runbook/release_progression", data["driver_requested_by"])
self.assertEqual("Release / Launchpad / 2026.04.18", data["sequence"]["target_scope_label"])
self.assertEqual(["确认发布包", "发灰度"], data["sequence"]["step_titles"])
self.assertEqual("release_launchpad", data["sequence"]["focus_ref"]["section"])
self.assertEqual("worker", data["sequence"]["action_focus_ref"]["mode"])
self.assertEqual("worker", data["driver_action_payload"]["focus_ref"]["mode"])
@patch("app.services.ops_service.get_ops_runbook")
def test_resolve_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
self,
mock_get_ops_runbook,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "node_onboarding",
"title": "纳管新节点",
"status": "warning",
"status_label": "有待接管节点",
"summary": "先补接入。",
"reason": "新增节点需要先纳管。",
"target_node_codes": ["mainland-worker-01"],
"secondary_label": "跑接管验收",
"secondary_action_code": "run_acceptance",
"secondary_action_payload": {
"node_code": "mainland-worker-01",
},
}
]
}
ok, message, data = resolve_ops_runbook_sequence(
"node_onboarding",
{
"secondary": True,
"requested_by": "test",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"auto_approve": True,
},
},
)
self.assertTrue(ok, message)
self.assertEqual("标准作业路径解析成功", message)
self.assertEqual("run_acceptance", data["driver_action_code"])
self.assertEqual("跑接管验收", data["driver_action_label"])
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
self.assertTrue(data["sequence"]["secondary"])
self.assertEqual(
{
"node_code": "mainland-worker-01",
"auto_approve": True,
},
data["driver_action_payload"],
)
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_uses_primary_action(
self,
mock_get_ops_runbook,
mock_execute_driver_action,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "可以直接发布。",
"reason": "launchpad 已就绪",
"target_node_codes": ["mainland-worker-01"],
"primary_action_code": "publish_latest_worker",
"primary_action_payload": {
"channel": "stable",
"release_id": 12,
},
"secondary_action_code": "focus_release_hub",
"secondary_action_payload": {
"release_id": 12,
},
}
]
}
mock_execute_driver_action.return_value = (
True,
"Release rollout 已创建",
{
"handled": True,
"mode": "smart-release-rollout",
},
)
ok, message, data = execute_ops_runbook_sequence(
"release_progression",
{
"requested_by": "test",
},
)
self.assertTrue(ok, message)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["handled"])
self.assertEqual("publish_latest_worker", data["driver_action_code"])
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
self.assertEqual("release_progression", data["sequence_key"])
self.assertEqual("版本发布与放量", data["sequence"]["title"])
mock_execute_driver_action.assert_called_once_with(
{
"action_code": "publish_latest_worker",
"node_codes": ["mainland-worker-01"],
"action_payload": {
"channel": "stable",
"release_id": 12,
},
"requested_by": "test/runbook/release_progression",
}
)
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
self,
mock_get_ops_runbook,
mock_execute_driver_action,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "node_onboarding",
"title": "纳管新节点",
"status": "warning",
"status_label": "有待接管节点",
"summary": "先补接入。",
"reason": "新增节点需要先纳管。",
"target_node_codes": ["mainland-worker-01"],
"primary_action_code": "bootstrap_run",
"primary_action_payload": {
"node_code": "mainland-worker-01",
},
"secondary_action_code": "run_acceptance",
"secondary_action_payload": {
"node_code": "mainland-worker-01",
},
}
]
}
mock_execute_driver_action.return_value = (
True,
"",
{
"handled": True,
"mode": "ui-intent",
},
)
ok, message, data = execute_ops_runbook_sequence(
"node_onboarding",
{
"secondary": True,
"requested_by": "test",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"auto_approve": True,
},
},
)
self.assertTrue(ok, message)
self.assertEqual("", message)
self.assertTrue(data["handled"])
self.assertEqual("run_acceptance", data["driver_action_code"])
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
self.assertTrue(data["sequence"]["secondary"])
mock_execute_driver_action.assert_called_once_with(
{
"action_code": "run_acceptance",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"node_code": "mainland-worker-01",
"auto_approve": True,
},
"requested_by": "test/runbook/node_onboarding",
}
)
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_returns_error_when_sequence_missing(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {"control_sequences": []}
ok, message, data = execute_ops_runbook_sequence("missing-sequence", {})
self.assertFalse(ok)
self.assertEqual("标准作业路径不存在", message)
self.assertEqual("missing-sequence", data["sequence_key"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,971 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_stack_diagnosis
class OpsServiceStackDiagnosisTests(unittest.TestCase):
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "fix_managed_nodes",
"title": "先补节点接管",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"mode": "registered",
"registered_paths_total": 24,
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
"ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
},
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
self.assertEqual("api-restart", issue["action_code"])
self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["api-restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
)
self.assertEqual(
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
payload["diagnosis"]["recommended_commands"]["stack-check"],
)
self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
self.assertTrue(
any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "bd6bcb2",
"route_surface": {
"mode": "registered",
"registered_paths_total": 12,
"surface_complete": False,
"missing_keys": [
"ops_node_onboarding_acceptance_execute",
"ops_node_onboarding_acceptance_preview",
],
"missing_paths": [
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
],
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
)
self.assertEqual("api-restart", issue["action_code"])
self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
issue["commands"],
)
self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
self.assertIn(
"ops_node_onboarding_acceptance_preview",
payload["runtime_build_info"]["route_surface_missing_keys"],
)
self.assertTrue(
any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("observability", decision["lane"])
self.assertEqual("scene-node-log", decision["primary_command_key"])
self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "等待执行 bootstrap。",
"stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "bootstrap_run",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
"recommended_target_node_code": "mainland-worker-01",
},
"worker_rollout_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-01",
"recovery_action": "bootstrap_run",
"recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
}
]
}
}
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertTrue(
any("当前第一动作已经收敛为 接入收口mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "handover_first_gap",
"title": "先补接管缺口",
"reason": "overview 只能先定位到接管缺口。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "agent_pending",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"status_label": "待补执行面",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"route_surface": {"surface_complete": True, "missing_keys": []},
"repository_capabilities": {},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual(
"mainland-worker-01",
payload["diagnosis"]["launchpad_recommended_target_node_code"],
)
self.assertEqual(
"签发接入工单",
payload["diagnosis"]["launchpad_recommended_recovery_label"],
)
self.assertIn(
"onboarding.bootstrap",
payload["diagnosis"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "blocked",
"summary_text": "还有节点未满足发布门禁。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待处理",
"recommended_action_code": "review_smart_rollout_preview",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("release", decision["lane"])
self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"release-launchpad http://127.0.0.1:8100 worker",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "attention",
"summary_text": "当前还有节点未完成接入。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "bootstrap_run",
"recommended_target_node_code": "mainland-worker-02",
"recommended_recovery_label": "待执行接入",
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 0,
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("focus-release-hub", decision["secondary_command_key"])
self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
decision["summary"],
)
self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_includes_first_handover_gap_details(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"status_counts": {"pending_bootstrap": 1},
"remote_access_state_counts": {"ssh_ready": 1},
"delivery_queue_state_counts": {"healthy": 1},
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "已签发有效 Token但目标节点尚未执行 bootstrap 片段。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"mainland-worker-01",
payload["managed_nodes"]["first_handover_gap_node_code"],
)
self.assertEqual(
"pending_bootstrap",
payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
)
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
)
self.assertIn("mainland-worker-01", issue["detail"])
self.assertTrue(
any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
)
self.assertTrue(
any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
mock_get_managed_node_handover.assert_called_once()
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "pbr-worker-logs-001",
"status": "attention",
"status_label": "待关注",
"focus_step_key": "worker_logs",
"focus_summary": "Worker 日志步骤仍未收口。",
"focus_ref": {
"kind": "playbook_run",
"run_code": "pbr-worker-logs-001",
"focus_step_key": "worker_logs",
},
}
]
}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "running",
"summary_text": "远端 Agent 已回传关键日志样本。",
"occurred_at": "2026-04-18 09:22:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 71,
"event_key": "job-event:71",
},
"ui_intent": {
"kind": "job_events",
"job_id": 71,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"ops_stack_diagnosis_contract",
payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
)
self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
top_item = payload["activity_stream"]["top_items"][0]
self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
self.assertEqual("job_events", top_item["ui_intent_kind"])
self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [
{
"label": "看参与节点日志",
"action_code": "open_worker_logs_participating",
"node_codes": ["mainland-worker-01"],
"payload": {},
}
],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"status": "busy",
}
],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
)
self.assertEqual("open_worker_logs_participating", issue["action_code"])
self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in issue["commands"]
)
)
self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in payload["diagnosis"]["quick_commands"]
)
)
self.assertTrue(
any(
"driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
for command in issue["commands"]
)
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "最近有任务回执异常。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "attention",
"job_status": "running",
"summary_text": "节点已接单执行,但开始回执未成功送达控制面。",
"occurred_at": "2026-04-18 10:22:00",
"job_code": "ops-agent-81",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"ui_intent": {
"kind": "job_events",
"job_id": 81,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
)
self.assertEqual("focus_latest_job_events", issue["action_code"])
self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
self.assertIn("temporary offline", issue["detail"])
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("ops_jobs", decision["lane"])
self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,177 @@
import unittest
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
class OpsTemplateServiceTests(unittest.TestCase):
def test_node_bootstrap_defaults_and_clamps_number(self) -> None:
ok, message, data = build_ops_template_payload(
"node.bootstrap",
{
"expires_in_hours": 9999,
},
)
self.assertTrue(ok, message)
self.assertEqual("ok", message)
self.assertEqual("", data["control_plane_base_url"])
self.assertEqual("/opt/domaincheck", data["root_dir"])
self.assertEqual(720, data["expires_in_hours"])
self.assertEqual("", data["node_region"])
self.assertEqual("", data["node_role"])
def test_deploy_release_custom_normalizes_lists_and_booleans(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.custom",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
"restart_services": "domaincheck-api\ndomaincheck-worker",
"health_check_urls": "",
"health_check_services": "",
"rollback_on_failure": "false",
"switch_current": "true",
},
)
self.assertTrue(ok, message)
self.assertEqual("ok", message)
self.assertEqual(["domaincheck-api", "domaincheck-worker"], data["restart_services"])
self.assertEqual([], data["health_check_urls"])
self.assertEqual([], data["health_check_services"])
self.assertFalse(data["rollback_on_failure"])
self.assertTrue(data["switch_current"])
self.assertEqual("/opt/domaincheck", data["install_root"])
def test_deploy_release_control_applies_defaults(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.control",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
)
self.assertTrue(ok, message)
self.assertEqual(
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
data["restart_services"],
)
self.assertEqual(["http://127.0.0.1:8100/health"], data["health_check_urls"])
self.assertEqual(
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
data["health_check_services"],
)
self.assertTrue(data["rollback_on_failure"])
self.assertTrue(data["switch_current"])
def test_deploy_release_missing_required_field_fails(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.custom",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "",
},
)
self.assertFalse(ok)
self.assertEqual("artifact_url 不能为空", message)
self.assertEqual({}, data)
def test_logs_collect_lines_are_clamped(self) -> None:
ok, message, data = build_ops_template_payload(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 9999,
},
)
self.assertTrue(ok, message)
self.assertEqual(500, data["lines"])
def test_delivery_queue_templates_normalize_and_require_expected_fields(self) -> None:
replay_ok, replay_message, replay_data = build_ops_template_payload(
"delivery.queue.replay",
{
"record_id": "evt-001",
"limit": 999,
"flush_after_replay": "false",
"reason": "确认状态已修复",
},
)
discard_ok, discard_message, discard_data = build_ops_template_payload(
"delivery.queue.discard",
{
"record_id": "evt-002",
"limit": 0,
"reason": "确认这条死信不再需要补发",
},
)
flush_ok, flush_message, flush_data = build_ops_template_payload(
"delivery.queue.flush",
{
"limit": 999,
},
)
discard_fail_ok, discard_fail_message, discard_fail_data = build_ops_template_payload(
"delivery.queue.discard",
{
"record_id": "evt-003",
"reason": "",
},
)
self.assertTrue(replay_ok, replay_message)
self.assertEqual(200, replay_data["limit"])
self.assertFalse(replay_data["flush_after_replay"])
self.assertEqual("evt-001", replay_data["record_id"])
self.assertTrue(discard_ok, discard_message)
self.assertEqual(1, discard_data["limit"])
self.assertEqual("确认这条死信不再需要补发", discard_data["reason"])
self.assertTrue(flush_ok, flush_message)
self.assertEqual(200, flush_data["limit"])
self.assertFalse(discard_fail_ok)
self.assertEqual("reason 不能为空", discard_fail_message)
self.assertEqual({}, discard_fail_data)
def test_diagnostics_and_service_control_templates_expose_ssh(self) -> None:
health_template = get_ops_action_template("health.snapshot")
logs_template = get_ops_action_template("logs.collect")
diagnostics_template = get_ops_action_template("diagnostics.collect")
service_status_template = get_ops_action_template("service.status")
restart_template = get_ops_action_template("service.restart")
restart_api_template = get_ops_action_template("runtime.restart_api")
stop_worker_template = get_ops_action_template("runtime.stop_worker")
deploy_control_template = get_ops_action_template("deploy.release.control")
deploy_worker_template = get_ops_action_template("deploy.release.worker")
deploy_custom_template = get_ops_action_template("deploy.release.custom")
queue_flush_template = get_ops_action_template("delivery.queue.flush")
queue_replay_template = get_ops_action_template("delivery.queue.replay")
queue_discard_template = get_ops_action_template("delivery.queue.discard")
self.assertIn("ssh", health_template["execution_modes"])
self.assertEqual("远端 Agent", health_template["default_execution_mode_label"])
self.assertEqual("SSH", next(item["label"] for item in health_template["execution_mode_options"] if item["value"] == "ssh"))
self.assertIn("ssh", logs_template["execution_modes"])
self.assertIn("ssh", diagnostics_template["execution_modes"])
self.assertIn("ssh", service_status_template["execution_modes"])
self.assertIn("ssh", restart_template["execution_modes"])
self.assertIn("ssh", restart_api_template["execution_modes"])
self.assertIn("ssh", stop_worker_template["execution_modes"])
self.assertEqual("控制面", get_ops_action_template("node.bootstrap")["default_execution_mode_label"])
self.assertIn("ssh", deploy_control_template["execution_modes"])
self.assertIn("ssh", deploy_worker_template["execution_modes"])
self.assertIn("ssh", deploy_custom_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_flush_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_replay_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_discard_template["execution_modes"])
self.assertEqual("远端 Agent", queue_flush_template["default_execution_mode_label"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,117 @@
from __future__ import annotations
import json
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
class PrepareFinalReleaseTests(unittest.TestCase):
def test_prepare_final_release_blocks_when_smoke_is_false_even_if_verify_passes(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
source_script = repo_root / "prepare_final_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
release_root = root / "release"
release_root.mkdir(parents=True, exist_ok=True)
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
(root / "package_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"mkdir -p \"$PWD/release\"\n"
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
"{\n"
" \"package_name\": \"domaincheck_release_test\",\n"
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
" \"sha256\": \"abc\",\n"
" \"smoke_test_ok\": false,\n"
" \"smoke_test_message\": \"skipped by env\"\n"
"}\n"
"EOF\n",
encoding="utf-8",
)
(root / "verify_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
encoding="utf-8",
)
os.chmod(root / "prepare_final_release.sh", 0o755)
os.chmod(root / "package_domain_release.sh", 0o755)
os.chmod(root / "verify_domain_release.sh", 0o755)
result = subprocess.run(
["bash", str(root / "prepare_final_release.sh")],
cwd=root,
check=True,
capture_output=True,
text=True,
)
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
self.assertFalse(payload["ok"])
self.assertEqual("blocked", payload["release_gate"]["decision"])
self.assertTrue(payload["release_gate"]["verify_ok"])
self.assertFalse(payload["release_gate"]["smoke_test_ok"])
self.assertIn("smoke_test_failed_or_skipped", payload["release_gate"]["blocked_reasons"])
self.assertIn('"ok": false', result.stdout)
def test_prepare_final_release_ready_when_verify_and_smoke_both_pass(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
source_script = repo_root / "prepare_final_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
release_root = root / "release"
release_root.mkdir(parents=True, exist_ok=True)
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
(root / "package_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"mkdir -p \"$PWD/release\"\n"
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
"{\n"
" \"package_name\": \"domaincheck_release_test\",\n"
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
" \"sha256\": \"abc\",\n"
" \"smoke_test_ok\": true,\n"
" \"smoke_test_message\": \"all checks passed (1/1)\"\n"
"}\n"
"EOF\n",
encoding="utf-8",
)
(root / "verify_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
encoding="utf-8",
)
os.chmod(root / "prepare_final_release.sh", 0o755)
os.chmod(root / "package_domain_release.sh", 0o755)
os.chmod(root / "verify_domain_release.sh", 0o755)
subprocess.run(
["bash", str(root / "prepare_final_release.sh")],
cwd=root,
check=True,
capture_output=True,
text=True,
)
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
self.assertTrue(payload["ok"])
self.assertEqual("ready", payload["release_gate"]["decision"])
self.assertEqual([], payload["release_gate"]["blocked_reasons"])
self.assertTrue(payload["release_gate"]["verify_ok"])
self.assertTrue(payload["release_gate"]["smoke_test_ok"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,29 @@
from __future__ import annotations
import unittest
from pathlib import Path
class ReleaseMetadataParityTests(unittest.TestCase):
def test_powershell_release_package_emits_archive_path_alias(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_text = (repo_root / "package_domain_release.ps1").read_text(encoding="utf-8")
self.assertIn("archive_path = $zipPath", script_text)
self.assertIn('"archive_path=$zipPath"', script_text)
def test_powershell_release_flow_prefers_archive_path_with_zip_fallback(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
prepare_text = (repo_root / "prepare_final_release.ps1").read_text(encoding="utf-8")
show_text = (repo_root / "show_latest_release.ps1").read_text(encoding="utf-8")
self.assertIn("$archivePath = if ($latest.archive_path) { $latest.archive_path } else { $latest.zip_path }", prepare_text)
self.assertIn('& (Join-Path $root "package_domain_release.ps1")', prepare_text)
self.assertIn('& (Join-Path $root "verify_domain_release.ps1")', prepare_text)
self.assertNotIn('powershell -ExecutionPolicy Bypass -File (Join-Path $root "package_domain_release.ps1")', prepare_text)
self.assertIn("$reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path }", show_text)
self.assertIn("$latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path }", show_text)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,19 @@
from __future__ import annotations
import unittest
from pathlib import Path
class ReleaseScriptParityTests(unittest.TestCase):
def test_shell_release_verifier_checks_cross_platform_delivery_scripts(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "verify_domain_release.sh"
script_text = script_path.read_text(encoding="utf-8")
self.assertIn('"scripts/package_domain_release.ps1"', script_text)
self.assertIn('"scripts/verify_domain_release.ps1"', script_text)
self.assertIn('"scripts/smoke_test_stack.ps1"', script_text)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,162 @@
from __future__ import annotations
import json
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
class ShowLatestReleaseTests(unittest.TestCase):
def test_show_latest_release_marks_stale_final_report_as_not_ok(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_new",
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
"sha256": "new-sha256",
}
final_report = {
"ok": True,
"latest_release": {
"package_name": "domaincheck_release_old",
"archive_path": str(release_root / "domaincheck_release_old.tar.gz"),
"sha256": "old-sha256",
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertFalse(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertFalse(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_old", payload["final_release_report_package_name"])
self.assertIn("package_name", payload["final_release_report_stale_reason"])
self.assertEqual("", payload["final_release_gate_decision"])
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
def test_show_latest_release_keeps_final_report_ok_when_it_matches_latest(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_new",
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
"sha256": "new-sha256",
}
final_report = {
"ok": True,
"latest_release": latest_payload,
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertTrue(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertTrue(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_new", payload["final_release_report_package_name"])
self.assertEqual("", payload["final_release_report_stale_reason"])
self.assertEqual("ready", payload["final_release_gate_decision"])
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
self.assertTrue(payload["final_release_gate_verify_ok"])
self.assertTrue(payload["final_release_gate_smoke_test_ok"])
def test_show_latest_release_accepts_zip_path_fallback(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_win",
"zip_path": str(release_root / "domaincheck_release_win.zip"),
"sha256": "zip-sha256",
}
final_report = {
"ok": True,
"latest_release": latest_payload,
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertTrue(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertTrue(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_win", payload["final_release_report_package_name"])
self.assertEqual("ready", payload["final_release_gate_decision"])
if __name__ == "__main__":
unittest.main()