feat: add ops center and node onboarding flow
This commit is contained in:
633
domain-api/app/api/routes/ops.py
Normal file
633
domain-api/app/api/routes/ops.py
Normal file
@@ -0,0 +1,633 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.schemas.common import ApiResponse
|
||||
from app.services.ops_agent_service import (
|
||||
build_managed_node_handover_bootstrap_plan,
|
||||
execute_managed_node_onboarding_bootstrap,
|
||||
execute_managed_node_onboarding_acceptance,
|
||||
execute_managed_node_onboarding_recovery,
|
||||
get_managed_node_delivery_queue,
|
||||
get_managed_node_handover,
|
||||
get_managed_node_onboarding,
|
||||
list_managed_node_delivery_queue_records,
|
||||
list_managed_nodes_with_agent_state,
|
||||
list_ops_job_events,
|
||||
preview_managed_node_onboarding_bootstrap,
|
||||
preview_managed_node_onboarding_acceptance,
|
||||
preview_managed_node_onboarding_recovery,
|
||||
request_managed_node_delivery_queue_action,
|
||||
summarize_ops_job_events,
|
||||
)
|
||||
from app.services.ops_job_service import (
|
||||
approve_ops_job,
|
||||
cancel_ops_job,
|
||||
create_ops_job,
|
||||
create_ops_job_batch,
|
||||
dispatch_ops_job,
|
||||
get_ops_job,
|
||||
list_ops_jobs,
|
||||
sync_managed_nodes_from_cluster,
|
||||
upsert_managed_node,
|
||||
)
|
||||
from app.services.ops_playbook_service import (
|
||||
cancel_ops_playbook_run,
|
||||
execute_ops_playbook,
|
||||
get_ops_playbook_run,
|
||||
get_ops_playbooks,
|
||||
get_recent_ops_playbook_runs,
|
||||
list_ops_playbook_run_events,
|
||||
preview_ops_playbook,
|
||||
rerun_ops_playbook_run,
|
||||
)
|
||||
from app.services.ops_policy_service import preview_ops_job_policy
|
||||
from app.services.ops_service import (
|
||||
execute_codex_action,
|
||||
execute_resolved_driver_action,
|
||||
get_ops_activity_stream,
|
||||
get_ops_codex_brief,
|
||||
get_ops_contract_detail,
|
||||
get_ops_contract_registry,
|
||||
get_ops_doctor_decision,
|
||||
get_ops_driver_feed,
|
||||
get_ops_go_live_summary,
|
||||
get_ops_go_live_signoff,
|
||||
get_ops_go_live_bundle,
|
||||
get_ops_go_live_review,
|
||||
get_ops_link_snapshot,
|
||||
get_ops_node_scene_log,
|
||||
get_ops_stack_diagnosis,
|
||||
execute_driver_action,
|
||||
preview_driver_action,
|
||||
resolve_codex_action,
|
||||
resolve_driver_action_request,
|
||||
resolve_ops_runbook_sequence,
|
||||
execute_ops_runbook_sequence,
|
||||
get_ops_blueprint,
|
||||
get_ops_capabilities,
|
||||
get_ops_inspection_overview,
|
||||
get_ops_overview,
|
||||
get_ops_runbook,
|
||||
)
|
||||
from app.services.ops_template_service import get_ops_action_templates
|
||||
|
||||
router = APIRouter(tags=["ops"])
|
||||
|
||||
|
||||
@router.get("/ops/overview", response_model=ApiResponse)
|
||||
def ops_overview() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_overview())
|
||||
|
||||
|
||||
@router.get("/ops/link-snapshot", response_model=ApiResponse)
|
||||
def ops_link_snapshot() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_link_snapshot())
|
||||
|
||||
|
||||
@router.get("/ops/stack-diagnosis", response_model=ApiResponse)
|
||||
def ops_stack_diagnosis(base_url: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_stack_diagnosis(base_url=base_url))
|
||||
|
||||
|
||||
@router.get("/ops/go-live-summary", response_model=ApiResponse)
|
||||
def ops_go_live_summary(base_url: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_go_live_summary(base_url=base_url))
|
||||
|
||||
|
||||
@router.get("/ops/doctor-decision", response_model=ApiResponse)
|
||||
def ops_doctor_decision(base_url: str = "", report_dir: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_doctor_decision(base_url=base_url, report_dir=report_dir))
|
||||
|
||||
|
||||
@router.get("/ops/go-live-signoff", response_model=ApiResponse)
|
||||
def ops_go_live_signoff(base_url: str = "", report_dir: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_go_live_signoff(base_url=base_url, report_dir=report_dir))
|
||||
|
||||
|
||||
@router.get("/ops/go-live-bundle", response_model=ApiResponse)
|
||||
def ops_go_live_bundle(base_url: str = "", report_dir: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_go_live_bundle(base_url=base_url, report_dir=report_dir))
|
||||
|
||||
|
||||
@router.get("/ops/go-live-review", response_model=ApiResponse)
|
||||
def ops_go_live_review(base_url: str = "", report_dir: str = "") -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_go_live_review(base_url=base_url, report_dir=report_dir))
|
||||
|
||||
|
||||
@router.get("/ops/inspection-overview", response_model=ApiResponse)
|
||||
def ops_inspection_overview(
|
||||
limit: int = 80,
|
||||
status: str = "",
|
||||
problem_kind: str = "",
|
||||
query: str = "",
|
||||
only_problem: bool = False,
|
||||
only_participating: bool = False,
|
||||
) -> ApiResponse:
|
||||
return ApiResponse(
|
||||
data=get_ops_inspection_overview(
|
||||
fetch_limit=limit,
|
||||
status=status,
|
||||
problem_kind=problem_kind,
|
||||
query=query,
|
||||
only_problem=only_problem,
|
||||
only_participating=only_participating,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ops/activity-stream", response_model=ApiResponse)
|
||||
def ops_activity_stream(
|
||||
limit: int = 18,
|
||||
kind: str = "",
|
||||
status: str = "",
|
||||
execution_mode: str = "",
|
||||
query: str = "",
|
||||
) -> ApiResponse:
|
||||
return ApiResponse(
|
||||
data=get_ops_activity_stream(
|
||||
limit=limit,
|
||||
kind=kind,
|
||||
status=status,
|
||||
execution_mode=execution_mode,
|
||||
query=query,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ops/driver-feed", response_model=ApiResponse)
|
||||
def ops_driver_feed() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_driver_feed())
|
||||
|
||||
|
||||
@router.get("/ops/codex-brief", response_model=ApiResponse)
|
||||
def ops_codex_brief() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_codex_brief())
|
||||
|
||||
|
||||
@router.get("/ops/capabilities", response_model=ApiResponse)
|
||||
def ops_capabilities() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_capabilities())
|
||||
|
||||
|
||||
@router.get("/ops/contracts", response_model=ApiResponse)
|
||||
def ops_contracts() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_contract_registry())
|
||||
|
||||
|
||||
@router.get("/ops/contracts/{contract_key}", response_model=ApiResponse)
|
||||
def ops_contract_detail(contract_key: str) -> ApiResponse:
|
||||
data = get_ops_contract_detail(contract_key)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="contract 不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/action-templates", response_model=ApiResponse)
|
||||
def ops_action_templates() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_action_templates())
|
||||
|
||||
|
||||
@router.get("/ops/playbooks", response_model=ApiResponse)
|
||||
def ops_playbooks() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_playbooks())
|
||||
|
||||
|
||||
@router.get("/ops/playbook-runs", response_model=ApiResponse)
|
||||
def ops_playbook_runs(
|
||||
limit: int = 12,
|
||||
status: str = "",
|
||||
group_key: str = "",
|
||||
query: str = "",
|
||||
) -> ApiResponse:
|
||||
return ApiResponse(
|
||||
data=get_recent_ops_playbook_runs(
|
||||
limit=limit,
|
||||
status=status,
|
||||
group_key=group_key,
|
||||
query=query,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ops/playbook-runs/{run_code}", response_model=ApiResponse)
|
||||
def ops_playbook_run_detail(run_code: str) -> ApiResponse:
|
||||
data = get_ops_playbook_run(run_code)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="playbook run 不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/playbook-runs/{run_code}/events", response_model=ApiResponse)
|
||||
def ops_playbook_run_events(
|
||||
run_code: str,
|
||||
limit: int = 80,
|
||||
step_key: str = "",
|
||||
node_code: str = "",
|
||||
) -> ApiResponse:
|
||||
data = list_ops_playbook_run_events(
|
||||
run_code,
|
||||
limit=limit,
|
||||
step_key=step_key,
|
||||
node_code=node_code,
|
||||
)
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.post("/ops/playbook-runs/{run_code}/rerun", response_model=ApiResponse)
|
||||
def ops_playbook_run_rerun(run_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
payload = payload or {}
|
||||
ok, message, data = rerun_ops_playbook_run(
|
||||
run_code,
|
||||
requested_by=str(payload.get("requested_by") or "api").strip() or "api",
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/playbook-runs/{run_code}/cancel", response_model=ApiResponse)
|
||||
def ops_playbook_run_cancel(run_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
payload = payload or {}
|
||||
ok, message, data = cancel_ops_playbook_run(
|
||||
run_code,
|
||||
cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api",
|
||||
reason=str(payload.get("reason") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/playbooks/preview", response_model=ApiResponse)
|
||||
def ops_playbook_preview(payload: dict) -> ApiResponse:
|
||||
ok, message, data = preview_ops_playbook(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/playbooks/execute", response_model=ApiResponse)
|
||||
def ops_playbook_execute(payload: dict) -> ApiResponse:
|
||||
ok, message, data = execute_ops_playbook(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.get("/ops/blueprint", response_model=ApiResponse)
|
||||
def ops_blueprint() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_blueprint())
|
||||
|
||||
|
||||
@router.get("/ops/runbook", response_model=ApiResponse)
|
||||
def ops_runbook() -> ApiResponse:
|
||||
return ApiResponse(data=get_ops_runbook())
|
||||
|
||||
|
||||
@router.post("/ops/runbook/sequences/{sequence_key}/resolve", response_model=ApiResponse)
|
||||
def ops_runbook_sequence_resolve(sequence_key: str, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = resolve_ops_runbook_sequence(sequence_key, payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/runbook/sequences/{sequence_key}/execute", response_model=ApiResponse)
|
||||
def ops_runbook_sequence_execute(sequence_key: str, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = execute_ops_runbook_sequence(sequence_key, payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.get("/ops/nodes", response_model=ApiResponse)
|
||||
def ops_nodes() -> ApiResponse:
|
||||
return ApiResponse(data=list_managed_nodes_with_agent_state())
|
||||
|
||||
|
||||
@router.get("/ops/nodes/{node_code}/handover", response_model=ApiResponse)
|
||||
def ops_node_handover(
|
||||
node_code: str,
|
||||
control_plane_base_url: str = "",
|
||||
root_dir: str = "",
|
||||
) -> ApiResponse:
|
||||
data = get_managed_node_handover(
|
||||
node_code,
|
||||
control_plane_base_url=control_plane_base_url,
|
||||
root_dir=root_dir,
|
||||
)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="节点不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/nodes/{node_code}/onboarding", response_model=ApiResponse)
|
||||
def ops_node_onboarding(
|
||||
node_code: str,
|
||||
control_plane_base_url: str = "",
|
||||
root_dir: str = "",
|
||||
) -> ApiResponse:
|
||||
data = get_managed_node_onboarding(
|
||||
node_code,
|
||||
control_plane_base_url=control_plane_base_url,
|
||||
root_dir=root_dir,
|
||||
)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="节点不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/acceptance/preview", response_model=ApiResponse)
|
||||
def ops_node_onboarding_acceptance_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = preview_managed_node_onboarding_acceptance(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/preview", response_model=ApiResponse)
|
||||
def ops_node_onboarding_bootstrap_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = preview_managed_node_onboarding_bootstrap(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/execute", response_model=ApiResponse)
|
||||
def ops_node_onboarding_bootstrap_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = execute_managed_node_onboarding_bootstrap(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/acceptance/execute", response_model=ApiResponse)
|
||||
def ops_node_onboarding_acceptance_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = execute_managed_node_onboarding_acceptance(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/recovery/preview", response_model=ApiResponse)
|
||||
def ops_node_onboarding_recovery_preview(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = preview_managed_node_onboarding_recovery(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/onboarding/recovery/execute", response_model=ApiResponse)
|
||||
def ops_node_onboarding_recovery_execute(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = execute_managed_node_onboarding_recovery(
|
||||
node_code=node_code,
|
||||
requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api",
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.get("/ops/nodes/{node_code}/scene-log", response_model=ApiResponse)
|
||||
def ops_node_scene_log(
|
||||
node_code: str,
|
||||
limit: int = 80,
|
||||
mode: str = "",
|
||||
) -> ApiResponse:
|
||||
data = get_ops_node_scene_log(node_code, limit=limit, mode=mode)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="节点不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/handover/bootstrap-plan", response_model=ApiResponse)
|
||||
def ops_node_handover_bootstrap_plan(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
normalized_payload = payload or {}
|
||||
ok, message, data = build_managed_node_handover_bootstrap_plan(
|
||||
node_code=node_code,
|
||||
issued_by=str(normalized_payload.get("issued_by") or "api").strip() or "api",
|
||||
expires_in_hours=int(normalized_payload.get("expires_in_hours") or 72),
|
||||
control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(normalized_payload.get("root_dir") or "").strip(),
|
||||
metadata=normalized_payload.get("metadata") or {},
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.get("/ops/nodes/{node_code}/delivery-queue", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue(node_code: str) -> ApiResponse:
|
||||
data = get_managed_node_delivery_queue(node_code)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="节点不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/nodes/{node_code}/delivery-queue/records", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue_records(
|
||||
node_code: str,
|
||||
state: str = "",
|
||||
limit: int = 50,
|
||||
) -> ApiResponse:
|
||||
data = list_managed_node_delivery_queue_records(
|
||||
node_code,
|
||||
state=state,
|
||||
limit=limit,
|
||||
)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="节点不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/delivery-queue/flush", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue_flush(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = request_managed_node_delivery_queue_action(
|
||||
node_code,
|
||||
"delivery.queue.flush",
|
||||
payload=payload or {},
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/delivery-queue/replay", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue_replay(node_code: str, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = request_managed_node_delivery_queue_action(
|
||||
node_code,
|
||||
"delivery.queue.replay",
|
||||
payload=payload or {},
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue_record_replay(
|
||||
node_code: str,
|
||||
record_id: str,
|
||||
payload: dict | None = None,
|
||||
) -> ApiResponse:
|
||||
ok, message, data = request_managed_node_delivery_queue_action(
|
||||
node_code,
|
||||
"delivery.queue.replay",
|
||||
payload=payload or {},
|
||||
record_id=record_id,
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", response_model=ApiResponse)
|
||||
def ops_node_delivery_queue_record_discard(
|
||||
node_code: str,
|
||||
record_id: str,
|
||||
payload: dict | None = None,
|
||||
) -> ApiResponse:
|
||||
ok, message, data = request_managed_node_delivery_queue_action(
|
||||
node_code,
|
||||
"delivery.queue.discard",
|
||||
payload=payload or {},
|
||||
record_id=record_id,
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes", response_model=ApiResponse)
|
||||
def ops_nodes_upsert(payload: dict) -> ApiResponse:
|
||||
ok, message, data = upsert_managed_node(payload)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/nodes/sync-from-cluster", response_model=ApiResponse)
|
||||
def ops_nodes_sync_from_cluster(dry_run: bool = False) -> ApiResponse:
|
||||
return ApiResponse(data=sync_managed_nodes_from_cluster(dry_run=dry_run))
|
||||
|
||||
|
||||
@router.get("/ops/jobs", response_model=ApiResponse)
|
||||
def ops_jobs(limit: int = 20, compact: bool = True, rollout_id: int = 0) -> ApiResponse:
|
||||
normalized_rollout_id = int(rollout_id or 0)
|
||||
jobs = list_ops_jobs(
|
||||
limit=limit,
|
||||
compact=compact,
|
||||
rollout_id=normalized_rollout_id if normalized_rollout_id > 0 else None,
|
||||
)
|
||||
page_status_counts: dict[str, int] = {}
|
||||
for item in jobs:
|
||||
item_status = str(item.get("status") or "").strip() or "unknown"
|
||||
page_status_counts[item_status] = int(page_status_counts.get(item_status, 0) or 0) + 1
|
||||
return ApiResponse(
|
||||
data={
|
||||
"jobs": jobs,
|
||||
"filters": {
|
||||
"limit": int(limit or 20),
|
||||
"compact": bool(compact),
|
||||
"rollout_id": normalized_rollout_id,
|
||||
},
|
||||
"page_summary": {
|
||||
"jobs_total": len(jobs),
|
||||
"status_counts": page_status_counts,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ops/jobs/{job_id}", response_model=ApiResponse)
|
||||
def ops_job_detail(job_id: int) -> ApiResponse:
|
||||
data = get_ops_job(job_id)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="任务不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/jobs/{job_id}/events", response_model=ApiResponse)
|
||||
def ops_job_events(job_id: int, limit: int = 50) -> ApiResponse:
|
||||
events = list_ops_job_events(job_id, limit=limit)
|
||||
return ApiResponse(
|
||||
data={
|
||||
"events": events,
|
||||
"summary": summarize_ops_job_events(events, job_id=job_id, limit=limit),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/ops/driver-actions/execute", response_model=ApiResponse)
|
||||
def ops_driver_action_execute(payload: dict) -> ApiResponse:
|
||||
ok, message, data = execute_driver_action(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/driver-actions/preview", response_model=ApiResponse)
|
||||
def ops_driver_action_preview(payload: dict) -> ApiResponse:
|
||||
ok, message, data = preview_driver_action(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/driver-actions/resolve", response_model=ApiResponse)
|
||||
def ops_driver_action_resolve(payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = resolve_driver_action_request(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/driver-actions/execute-resolved", response_model=ApiResponse)
|
||||
def ops_driver_action_execute_resolved(payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = execute_resolved_driver_action(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/codex-actions/resolve", response_model=ApiResponse)
|
||||
def ops_codex_action_resolve(payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = resolve_codex_action(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/codex-actions/execute", response_model=ApiResponse)
|
||||
def ops_codex_action_execute(payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = execute_codex_action(payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/jobs", response_model=ApiResponse)
|
||||
def ops_jobs_create(payload: dict) -> ApiResponse:
|
||||
ok, message, data = create_ops_job(payload)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/jobs/batch", response_model=ApiResponse)
|
||||
def ops_jobs_create_batch(payload: dict) -> ApiResponse:
|
||||
ok, message, data = create_ops_job_batch(payload)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/policy/preview", response_model=ApiResponse)
|
||||
def ops_policy_preview(payload: dict) -> ApiResponse:
|
||||
return ApiResponse(data=preview_ops_job_policy(payload))
|
||||
|
||||
|
||||
@router.post("/ops/jobs/{job_id}/approve", response_model=ApiResponse)
|
||||
def ops_job_approve(job_id: int, payload: dict | None = None) -> ApiResponse:
|
||||
payload = payload or {}
|
||||
ok, message, data = approve_ops_job(job_id, approved_by=str(payload.get("approved_by") or "api").strip() or "api")
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/jobs/{job_id}/cancel", response_model=ApiResponse)
|
||||
def ops_job_cancel(job_id: int, payload: dict | None = None) -> ApiResponse:
|
||||
payload = payload or {}
|
||||
ok, message, data = cancel_ops_job(
|
||||
job_id,
|
||||
cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api",
|
||||
reason=str(payload.get("reason") or "").strip(),
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/jobs/{job_id}/dispatch", response_model=ApiResponse)
|
||||
def ops_job_dispatch(job_id: int) -> ApiResponse:
|
||||
ok, message, data = dispatch_ops_job(job_id)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
90
domain-api/app/api/routes/ops_agent.py
Normal file
90
domain-api/app/api/routes/ops_agent.py
Normal file
@@ -0,0 +1,90 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Header
|
||||
|
||||
from app.schemas.common import ApiResponse
|
||||
from app.services.ops_agent_service import (
|
||||
agent_append_job_event,
|
||||
agent_complete_job,
|
||||
agent_heartbeat,
|
||||
agent_mark_job_started,
|
||||
agent_pull_jobs,
|
||||
agent_register,
|
||||
build_node_agent_bootstrap_plan,
|
||||
issue_node_agent_token,
|
||||
)
|
||||
|
||||
router = APIRouter(tags=["ops-agent"])
|
||||
|
||||
|
||||
def _resolve_agent_token(x_domaincheck_agent_token: Optional[str]) -> str:
|
||||
return str(x_domaincheck_agent_token or "").strip()
|
||||
|
||||
|
||||
def _build_agent_response(ok: bool, message: str, data: object) -> ApiResponse:
|
||||
detail_code = None
|
||||
if isinstance(data, dict):
|
||||
detail_code = str(data.get("detail_code") or "").strip() or None
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data, detail_code=detail_code)
|
||||
|
||||
|
||||
@router.post("/ops/agent/tokens", response_model=ApiResponse)
|
||||
def ops_agent_issue_token(payload: dict) -> ApiResponse:
|
||||
ok, message, data = issue_node_agent_token(
|
||||
node_code=str(payload.get("node_code") or "").strip(),
|
||||
issued_by=str(payload.get("issued_by") or "api").strip() or "api",
|
||||
expires_in_hours=int(payload.get("expires_in_hours") or 72),
|
||||
metadata=payload.get("metadata") or {},
|
||||
)
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/bootstrap-plan", response_model=ApiResponse)
|
||||
def ops_agent_bootstrap_plan(payload: dict) -> ApiResponse:
|
||||
ok, message, data = build_node_agent_bootstrap_plan(
|
||||
node_code=str(payload.get("node_code") or "").strip(),
|
||||
node_region=str(payload.get("node_region") or "mainland").strip() or "mainland",
|
||||
node_role=str(payload.get("node_role") or "worker").strip() or "worker",
|
||||
issued_by=str(payload.get("issued_by") or "api").strip() or "api",
|
||||
expires_in_hours=int(payload.get("expires_in_hours") or 72),
|
||||
control_plane_base_url=str(payload.get("control_plane_base_url") or "").strip(),
|
||||
root_dir=str(payload.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck",
|
||||
metadata=payload.get("metadata") or {},
|
||||
)
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/register", response_model=ApiResponse)
|
||||
def ops_agent_register(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_register(payload, token=_resolve_agent_token(x_domaincheck_agent_token))
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/heartbeat", response_model=ApiResponse)
|
||||
def ops_agent_heartbeat(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_heartbeat(payload, token=_resolve_agent_token(x_domaincheck_agent_token))
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/pull", response_model=ApiResponse)
|
||||
def ops_agent_pull(payload: dict, limit: int = 1, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_pull_jobs(payload, token=_resolve_agent_token(x_domaincheck_agent_token), limit=limit)
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/jobs/{job_id}/start", response_model=ApiResponse)
|
||||
def ops_agent_job_start(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_mark_job_started(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/jobs/{job_id}/complete", response_model=ApiResponse)
|
||||
def ops_agent_job_complete(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_complete_job(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
|
||||
return _build_agent_response(ok, message, data)
|
||||
|
||||
|
||||
@router.post("/ops/agent/jobs/{job_id}/events", response_model=ApiResponse)
|
||||
def ops_agent_job_event(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
|
||||
ok, message, data = agent_append_job_event(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))
|
||||
return _build_agent_response(ok, message, data)
|
||||
175
domain-api/app/api/routes/ops_release.py
Normal file
175
domain-api/app/api/routes/ops_release.py
Normal file
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.schemas.common import ApiResponse
|
||||
from app.services.ops_release_service import (
|
||||
advance_release_rollout,
|
||||
activate_release,
|
||||
create_release,
|
||||
create_release_and_smart_rollout_from_latest_package,
|
||||
create_release_from_latest_package,
|
||||
create_release_rollout,
|
||||
create_smart_release_rollout,
|
||||
get_latest_release_package_metadata,
|
||||
get_release_launchpad,
|
||||
get_latest_release,
|
||||
get_release_package_archive_path,
|
||||
get_release_package_sha256_path,
|
||||
get_release,
|
||||
get_release_rollout,
|
||||
list_release_rollout_jobs,
|
||||
list_release_rollouts,
|
||||
list_releases,
|
||||
preview_release_from_latest_package,
|
||||
preview_smart_release_rollout,
|
||||
preview_smart_release_rollout_from_latest_package,
|
||||
)
|
||||
|
||||
router = APIRouter(tags=["ops-release"])
|
||||
|
||||
|
||||
@router.get("/ops/releases", response_model=ApiResponse)
|
||||
def ops_releases(limit: int = 20, channel: str | None = None) -> ApiResponse:
|
||||
return ApiResponse(data={"releases": list_releases(limit=limit, channel=channel)})
|
||||
|
||||
|
||||
@router.get("/ops/releases/latest", response_model=ApiResponse)
|
||||
def ops_releases_latest(channel: str = "stable") -> ApiResponse:
|
||||
return ApiResponse(data=get_latest_release(channel=channel))
|
||||
|
||||
|
||||
@router.get("/ops/releases/package-metadata/latest", response_model=ApiResponse)
|
||||
def ops_releases_package_metadata_latest() -> ApiResponse:
|
||||
return ApiResponse(data=get_latest_release_package_metadata())
|
||||
|
||||
|
||||
@router.get("/ops/releases/launchpad", response_model=ApiResponse)
|
||||
def ops_releases_launchpad(request: Request, channel: str = "stable") -> ApiResponse:
|
||||
derived_base_url = str(request.base_url).rstrip("/")
|
||||
return ApiResponse(data=get_release_launchpad(control_plane_base_url=derived_base_url, channel=channel))
|
||||
|
||||
|
||||
@router.get("/ops/releases/packages/{package_name}/download")
|
||||
def ops_release_package_download(package_name: str):
|
||||
try:
|
||||
archive_path = get_release_package_archive_path(package_name)
|
||||
except ValueError:
|
||||
return ApiResponse(code=1, message="发布包名称不合法", data={})
|
||||
if not archive_path:
|
||||
return ApiResponse(code=1, message="发布包不存在", data={})
|
||||
media_type = "application/octet-stream"
|
||||
archive_name = archive_path.name.lower()
|
||||
if archive_name.endswith(".tar.gz"):
|
||||
media_type = "application/gzip"
|
||||
elif archive_name.endswith(".zip"):
|
||||
media_type = "application/zip"
|
||||
return FileResponse(path=archive_path, filename=archive_path.name, media_type=media_type)
|
||||
|
||||
|
||||
@router.get("/ops/releases/packages/{package_name}/sha256")
|
||||
def ops_release_package_sha256(package_name: str):
|
||||
try:
|
||||
sha256_path = get_release_package_sha256_path(package_name)
|
||||
except ValueError:
|
||||
return ApiResponse(code=1, message="发布包名称不合法", data={})
|
||||
if not sha256_path:
|
||||
return ApiResponse(code=1, message="发布包校验文件不存在", data={})
|
||||
return FileResponse(path=sha256_path, filename=sha256_path.name, media_type="text/plain; charset=utf-8")
|
||||
|
||||
|
||||
@router.get("/ops/releases/{release_id}", response_model=ApiResponse)
|
||||
def ops_releases_detail(release_id: int) -> ApiResponse:
|
||||
data = get_release(release_id)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="Release 不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases", response_model=ApiResponse)
|
||||
def ops_releases_create(payload: dict) -> ApiResponse:
|
||||
ok, message, data = create_release(payload)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/from-package/latest", response_model=ApiResponse)
|
||||
def ops_releases_create_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse:
|
||||
derived_base_url = str(request.base_url).rstrip("/")
|
||||
ok, message, data = create_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/from-package/latest/preview", response_model=ApiResponse)
|
||||
def ops_releases_preview_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse:
|
||||
derived_base_url = str(request.base_url).rstrip("/")
|
||||
ok, message, data = preview_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/from-package/latest/smart-rollout", response_model=ApiResponse)
|
||||
def ops_releases_create_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse:
|
||||
derived_base_url = str(request.base_url).rstrip("/")
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
payload or {},
|
||||
control_plane_base_url=derived_base_url,
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/from-package/latest/smart-rollout-preview", response_model=ApiResponse)
|
||||
def ops_releases_preview_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse:
|
||||
derived_base_url = str(request.base_url).rstrip("/")
|
||||
ok, message, data = preview_smart_release_rollout_from_latest_package(
|
||||
payload or {},
|
||||
control_plane_base_url=derived_base_url,
|
||||
)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/{release_id}/activate", response_model=ApiResponse)
|
||||
def ops_releases_activate(release_id: int) -> ApiResponse:
|
||||
ok, message, data = activate_release(release_id)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/{release_id}/smart-rollout-preview", response_model=ApiResponse)
|
||||
def ops_releases_smart_rollout_preview(release_id: int, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = preview_smart_release_rollout(release_id, payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/{release_id}/smart-rollout", response_model=ApiResponse)
|
||||
def ops_releases_smart_rollout(release_id: int, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = create_smart_release_rollout(release_id, payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.get("/ops/releases/{release_id}/rollouts", response_model=ApiResponse)
|
||||
def ops_release_rollouts(release_id: int, limit: int = 20) -> ApiResponse:
|
||||
return ApiResponse(data={"rollouts": list_release_rollouts(release_id=release_id, limit=limit)})
|
||||
|
||||
|
||||
@router.get("/ops/rollouts/{rollout_id}", response_model=ApiResponse)
|
||||
def ops_rollout_detail(rollout_id: int) -> ApiResponse:
|
||||
data = get_release_rollout(rollout_id)
|
||||
if not data:
|
||||
return ApiResponse(code=1, message="Rollout 不存在", data={})
|
||||
return ApiResponse(data=data)
|
||||
|
||||
|
||||
@router.get("/ops/rollouts/{rollout_id}/jobs", response_model=ApiResponse)
|
||||
def ops_rollout_jobs(rollout_id: int, limit: int = 200) -> ApiResponse:
|
||||
return ApiResponse(data={"jobs": list_release_rollout_jobs(rollout_id, limit=limit)})
|
||||
|
||||
|
||||
@router.post("/ops/rollouts/{rollout_id}/advance", response_model=ApiResponse)
|
||||
def ops_rollout_advance(rollout_id: int, payload: dict | None = None) -> ApiResponse:
|
||||
ok, message, data = advance_release_rollout(rollout_id, payload or {})
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
|
||||
|
||||
@router.post("/ops/releases/{release_id}/rollouts", response_model=ApiResponse)
|
||||
def ops_release_rollouts_create(release_id: int, payload: dict) -> ApiResponse:
|
||||
ok, message, data = create_release_rollout(release_id, payload)
|
||||
return ApiResponse(code=0 if ok else 1, message=message, data=data)
|
||||
@@ -3,6 +3,7 @@ from typing import Optional
|
||||
from fastapi import APIRouter, Header
|
||||
|
||||
from app.schemas.common import ApiResponse
|
||||
from app.services.build_info_service import get_runtime_build_info
|
||||
from app.services.cluster_runtime_service import get_cluster_snapshot
|
||||
from app.services.debug_event_service import get_debug_diagnosis, get_debug_event_overview, get_debug_handoff_report, ingest_debug_event, list_debug_events
|
||||
from app.services.runtime_control_service import runtime_action
|
||||
@@ -37,6 +38,11 @@ def runtime_cluster() -> ApiResponse:
|
||||
return ApiResponse(data=get_cluster_snapshot())
|
||||
|
||||
|
||||
@router.get("/runtime/build-info", response_model=ApiResponse)
|
||||
def runtime_build_info() -> ApiResponse:
|
||||
return ApiResponse(data=get_runtime_build_info())
|
||||
|
||||
|
||||
@router.get("/runtime/sync-summary", response_model=ApiResponse)
|
||||
def runtime_sync_summary() -> ApiResponse:
|
||||
return ApiResponse(data=get_sync_summary())
|
||||
|
||||
@@ -42,6 +42,13 @@ class Settings(BaseSettings):
|
||||
sync_shared_token: str = ""
|
||||
sync_batch_size: int = 200
|
||||
sync_poll_interval_seconds: int = 30
|
||||
build_manifest_path: str = ""
|
||||
build_commit_sha: str = ""
|
||||
build_commit_ref: str = ""
|
||||
build_generated_at: str = ""
|
||||
build_package_name: str = ""
|
||||
build_checksum: str = ""
|
||||
build_source_label: str = ""
|
||||
|
||||
@property
|
||||
def cors_origins_list(self) -> list[str]:
|
||||
|
||||
@@ -3,9 +3,13 @@ import threading
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, sensitive_words
|
||||
from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, ops, ops_agent, ops_release, sensitive_words
|
||||
from app.core.config import settings as app_settings
|
||||
from app.services.build_info_service import get_runtime_build_info, remember_registered_route_paths
|
||||
from app.services.cluster_runtime_service import ensure_runtime_schema, register_local_control_heartbeat
|
||||
from app.services.ops_agent_service import ensure_ops_agent_schema
|
||||
from app.services.ops_job_service import ensure_ops_schema
|
||||
from app.services.ops_release_service import ensure_ops_release_schema
|
||||
_heartbeat_stop_event = threading.Event()
|
||||
|
||||
|
||||
@@ -36,6 +40,10 @@ app.add_middleware(
|
||||
@app.on_event("startup")
|
||||
def on_startup() -> None:
|
||||
ensure_runtime_schema()
|
||||
ensure_ops_schema()
|
||||
ensure_ops_agent_schema()
|
||||
ensure_ops_release_schema()
|
||||
remember_registered_route_paths(route.path for route in app.routes)
|
||||
register_local_control_heartbeat()
|
||||
_heartbeat_stop_event.clear()
|
||||
threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start()
|
||||
@@ -56,6 +64,7 @@ def health() -> dict:
|
||||
"worker_mode": app_settings.worker_mode,
|
||||
"api_host": app_settings.api_host,
|
||||
"api_port": app_settings.api_port,
|
||||
"build": get_runtime_build_info(route_paths=(route.path for route in app.routes)),
|
||||
}
|
||||
|
||||
|
||||
@@ -69,4 +78,8 @@ app.include_router(exports.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(logs.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(runtime.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(juming.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(ops.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(ops_agent.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(ops_release.router, prefix=app_settings.api_prefix)
|
||||
app.include_router(sensitive_words.router, prefix=app_settings.api_prefix)
|
||||
remember_registered_route_paths(route.path for route in app.routes)
|
||||
|
||||
991
domain-api/app/node_agent.py
Normal file
991
domain-api/app/node_agent.py
Normal file
@@ -0,0 +1,991 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from app.services.ops_action_executor_core import (
|
||||
build_service_name_map,
|
||||
execute_structured_action,
|
||||
supports_structured_action,
|
||||
)
|
||||
from app.services.ops_release_executor_core import (
|
||||
execute_release_action,
|
||||
normalize_release_health_check_services as _release_normalize_health_check_services,
|
||||
normalize_text_list as _release_normalize_text_list,
|
||||
)
|
||||
|
||||
|
||||
CONTROL_PLANE_BASE_URL = str(os.getenv("OPS_CONTROL_PLANE_BASE_URL", "")).strip().rstrip("/")
|
||||
AGENT_TOKEN = str(os.getenv("OPS_AGENT_TOKEN", "")).strip()
|
||||
NODE_CODE = str(os.getenv("NODE_CODE", "")).strip()
|
||||
NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland"
|
||||
NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker"
|
||||
AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5))
|
||||
WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker"
|
||||
API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api"
|
||||
SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent"
|
||||
NODE_AGENT_SERVICE_NAME = str(os.getenv("NODE_AGENT_SERVICE_NAME", "domaincheck-node-agent")).strip() or "domaincheck-node-agent"
|
||||
AGENT_VERSION = "0.1.0"
|
||||
_APP_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
_PROJECT_DIR = os.path.dirname(_APP_DIR)
|
||||
_DEFAULT_QUEUE_DIR = os.path.join(_PROJECT_DIR, "runtime", "node-agent-queue", NODE_CODE or "unbound")
|
||||
AGENT_QUEUE_DIR = str(os.getenv("OPS_AGENT_QUEUE_DIR", _DEFAULT_QUEUE_DIR)).strip() or _DEFAULT_QUEUE_DIR
|
||||
AGENT_QUEUE_FLUSH_LIMIT = max(1, int(os.getenv("OPS_AGENT_QUEUE_FLUSH_LIMIT", "20") or 20))
|
||||
_PERMANENT_DELIVERY_DETAIL_CODES = {
|
||||
"agent_node_code_required",
|
||||
"ops_job_not_owned_by_agent",
|
||||
"ops_job_invalid_status",
|
||||
}
|
||||
_LAST_QUEUE_FLUSH_SUMMARY = {
|
||||
"scanned": 0,
|
||||
"delivered": 0,
|
||||
"deferred": 0,
|
||||
"dead_letter": 0,
|
||||
"last_flush_at": "",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_text_list(raw_value: object) -> list[str]:
|
||||
"""Keep node-agent payload normalization aligned with release executor helpers."""
|
||||
return _release_normalize_text_list(raw_value)
|
||||
|
||||
|
||||
def _normalize_release_health_check_services(
|
||||
payload: dict | None,
|
||||
restart_services: list[str] | None,
|
||||
) -> list[str]:
|
||||
"""Backward-compatible wrapper used by node-agent tests and payload shaping."""
|
||||
return _release_normalize_health_check_services(
|
||||
dict(payload or {}),
|
||||
list(restart_services or []),
|
||||
default_api_service_name=API_SERVICE_NAME,
|
||||
)
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
print(f"{datetime.now().isoformat(sep=' ', timespec='seconds')} [node-agent] {message}", flush=True)
|
||||
|
||||
|
||||
def _json_env(name: str, fallback: object) -> object:
|
||||
raw = str(os.getenv(name, "")).strip()
|
||||
if not raw:
|
||||
return fallback
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception:
|
||||
return fallback
|
||||
|
||||
|
||||
AGENT_CAPABILITIES = _json_env(
|
||||
"OPS_AGENT_CAPABILITIES",
|
||||
[
|
||||
"service.start",
|
||||
"service.stop",
|
||||
"service.restart",
|
||||
"service.status",
|
||||
"runtime.start_worker",
|
||||
"runtime.stop_worker",
|
||||
"runtime.restart_api",
|
||||
"runtime.start_sync_agent",
|
||||
"runtime.stop_sync_agent",
|
||||
"health.snapshot",
|
||||
"logs.collect",
|
||||
"diagnostics.collect",
|
||||
"delivery.queue.flush",
|
||||
"delivery.queue.replay",
|
||||
"delivery.queue.discard",
|
||||
"deploy.release",
|
||||
],
|
||||
)
|
||||
AGENT_LABELS = _json_env("OPS_AGENT_LABELS", {})
|
||||
|
||||
|
||||
class AgentResponseError(RuntimeError):
|
||||
def __init__(self, message: str, *, detail_code: str = "", payload: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.detail_code = str(detail_code or "").strip()
|
||||
self.payload = dict(payload or {})
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
return datetime.now().isoformat(sep=" ", timespec="seconds")
|
||||
|
||||
|
||||
def _queue_pending_dir() -> str:
|
||||
return os.path.join(AGENT_QUEUE_DIR, "pending")
|
||||
|
||||
|
||||
def _queue_dead_letter_dir() -> str:
|
||||
return os.path.join(AGENT_QUEUE_DIR, "dead-letter")
|
||||
|
||||
|
||||
def _queue_discarded_dir() -> str:
|
||||
return os.path.join(AGENT_QUEUE_DIR, "discarded")
|
||||
|
||||
|
||||
def _ensure_queue_dirs() -> None:
|
||||
for path in (AGENT_QUEUE_DIR, _queue_pending_dir(), _queue_dead_letter_dir(), _queue_discarded_dir()):
|
||||
os.makedirs(path, exist_ok=True)
|
||||
|
||||
|
||||
def _write_json_file(path: str, payload: dict) -> None:
|
||||
temp_path = f"{path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||||
os.replace(temp_path, path)
|
||||
|
||||
|
||||
def _read_json_file(path: str) -> dict:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def _new_request_id(prefix: str, job_id: int) -> str:
|
||||
normalized_prefix = str(prefix or "request").strip() or "request"
|
||||
return f"{normalized_prefix}-{int(job_id or 0)}-{int(time.time() * 1000)}-{uuid4().hex[:10]}"
|
||||
|
||||
|
||||
def _queue_file_names(directory: str) -> list[str]:
|
||||
_ensure_queue_dirs()
|
||||
try:
|
||||
return sorted(file_name for file_name in os.listdir(directory) if file_name.endswith(".json"))
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
|
||||
def _queue_head_record(directory: str) -> dict:
|
||||
file_names = _queue_file_names(directory)
|
||||
if not file_names:
|
||||
return {}
|
||||
file_path = os.path.join(directory, file_names[0])
|
||||
try:
|
||||
return _read_json_file(file_path)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _queue_record_id(record: dict, *, file_name: str = "") -> str:
|
||||
request_id = str((record or {}).get("request_id") or "").strip()
|
||||
if request_id:
|
||||
return request_id
|
||||
normalized_file_name = str(file_name or "").strip()
|
||||
if normalized_file_name.endswith(".json"):
|
||||
return normalized_file_name[:-5]
|
||||
return normalized_file_name
|
||||
|
||||
|
||||
def _queue_entries(state: str) -> list[dict]:
|
||||
normalized_state = str(state or "").strip()
|
||||
if normalized_state == "pending":
|
||||
directory = _queue_pending_dir()
|
||||
elif normalized_state == "dead_letter":
|
||||
directory = _queue_dead_letter_dir()
|
||||
else:
|
||||
return []
|
||||
|
||||
entries: list[dict] = []
|
||||
for file_name in _queue_file_names(directory):
|
||||
file_path = os.path.join(directory, file_name)
|
||||
try:
|
||||
record = _read_json_file(file_path)
|
||||
except Exception:
|
||||
continue
|
||||
entries.append(
|
||||
{
|
||||
"state": normalized_state,
|
||||
"file_name": file_name,
|
||||
"file_path": file_path,
|
||||
"record_id": _queue_record_id(record, file_name=file_name),
|
||||
"record": dict(record or {}),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def _matches_delivery_selector(entry: dict, selector: dict, *, allowed_states: set[str] | None = None) -> bool:
|
||||
normalized_selector = dict(selector or {})
|
||||
record = dict(entry.get("record") or {})
|
||||
state = str(entry.get("state") or "").strip()
|
||||
if allowed_states and state not in allowed_states:
|
||||
return False
|
||||
|
||||
selected_state = str(normalized_selector.get("state") or "").strip()
|
||||
if selected_state and state != selected_state:
|
||||
return False
|
||||
|
||||
selected_record_id = str(
|
||||
normalized_selector.get("record_id")
|
||||
or normalized_selector.get("request_id")
|
||||
or ""
|
||||
).strip()
|
||||
if selected_record_id and str(entry.get("record_id") or "").strip() != selected_record_id:
|
||||
return False
|
||||
|
||||
selected_request_kind = str(normalized_selector.get("request_kind") or "").strip()
|
||||
if selected_request_kind and str(record.get("kind") or "").strip() != selected_request_kind:
|
||||
return False
|
||||
|
||||
selected_detail_code = str(normalized_selector.get("detail_code") or "").strip()
|
||||
if selected_detail_code and str(record.get("last_detail_code") or "").strip() != selected_detail_code:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _trim_queue_preview(records: list[dict], *, limit: int = 5) -> list[dict]:
|
||||
safe_limit = max(1, int(limit or 5))
|
||||
preview: list[dict] = []
|
||||
for entry in list(records or [])[:safe_limit]:
|
||||
record = dict(entry.get("record") or {})
|
||||
preview.append(
|
||||
{
|
||||
"record_id": str(entry.get("record_id") or "").strip(),
|
||||
"state": str(entry.get("state") or "").strip(),
|
||||
"request_kind": str(record.get("kind") or "").strip(),
|
||||
"request_id": str(record.get("request_id") or "").strip(),
|
||||
"detail_code": str(record.get("last_detail_code") or "").strip(),
|
||||
"created_at": str(record.get("created_at") or "").strip(),
|
||||
"updated_at": str(record.get("updated_at") or "").strip(),
|
||||
}
|
||||
)
|
||||
return preview
|
||||
|
||||
|
||||
def _normalize_queue_limit(raw_value: object, *, default: int = 20, minimum: int = 1, maximum: int = 200) -> int:
|
||||
try:
|
||||
value = int(raw_value or default)
|
||||
except Exception:
|
||||
value = int(default)
|
||||
return max(minimum, min(maximum, value))
|
||||
|
||||
|
||||
def _replay_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_payload = dict(payload or {})
|
||||
selector = dict(normalized_payload.get("selector") or {})
|
||||
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
|
||||
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
|
||||
flush_after_replay = bool(normalized_payload.get("flush_after_replay", True))
|
||||
reason = str(normalized_payload.get("reason") or "").strip()
|
||||
|
||||
matched_entries = [
|
||||
entry
|
||||
for entry in _queue_entries("dead_letter")
|
||||
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
|
||||
][:limit]
|
||||
if not matched_entries:
|
||||
return False, "未找到符合条件的死信记录", {
|
||||
"selector": selector,
|
||||
"limit": limit,
|
||||
"queue": _delivery_queue_snapshot(),
|
||||
}
|
||||
|
||||
replayed_total = 0
|
||||
for entry in matched_entries:
|
||||
record = dict(entry.get("record") or {})
|
||||
record["updated_at"] = _now_text()
|
||||
record["replay_count"] = int(record.get("replay_count") or 0) + 1
|
||||
record["last_replay_at"] = _now_text()
|
||||
if reason:
|
||||
record["last_replay_reason"] = reason
|
||||
record.pop("dead_letter_at", None)
|
||||
record.pop("dead_letter_reason", None)
|
||||
_store_pending_delivery(record)
|
||||
try:
|
||||
os.remove(str(entry.get("file_path") or ""))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
replayed_total += 1
|
||||
|
||||
flush_summary = {}
|
||||
if flush_after_replay and replayed_total > 0:
|
||||
flush_summary = _flush_delivery_queue(limit=replayed_total)
|
||||
|
||||
result = {
|
||||
"selector": selector,
|
||||
"limit": limit,
|
||||
"replayed_total": replayed_total,
|
||||
"flush_after_replay": flush_after_replay,
|
||||
"flush_summary": flush_summary,
|
||||
"matched_records": _trim_queue_preview(matched_entries),
|
||||
"queue": _delivery_queue_snapshot(),
|
||||
}
|
||||
return True, f"已重放 {replayed_total} 条死信记录", result
|
||||
|
||||
|
||||
def _store_discarded_delivery(record: dict, *, discarded_by: str = "", reason: str = "") -> str:
|
||||
_ensure_queue_dirs()
|
||||
discarded_record = dict(record or {})
|
||||
discarded_record["discarded_at"] = _now_text()
|
||||
discarded_record["discarded_by"] = str(discarded_by or "").strip()
|
||||
discarded_record["discard_reason"] = str(reason or "").strip()
|
||||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||||
file_path = os.path.join(_queue_discarded_dir(), file_name)
|
||||
_write_json_file(file_path, discarded_record)
|
||||
return file_path
|
||||
|
||||
|
||||
def _discard_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_payload = dict(payload or {})
|
||||
selector = dict(normalized_payload.get("selector") or {})
|
||||
selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter"
|
||||
limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20)
|
||||
discarded_by = str(normalized_payload.get("discarded_by") or "").strip()
|
||||
reason = str(normalized_payload.get("reason") or "").strip()
|
||||
|
||||
matched_entries = [
|
||||
entry
|
||||
for entry in _queue_entries("dead_letter")
|
||||
if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"})
|
||||
][:limit]
|
||||
if not matched_entries:
|
||||
return False, "未找到符合条件的死信记录", {
|
||||
"selector": selector,
|
||||
"limit": limit,
|
||||
"queue": _delivery_queue_snapshot(),
|
||||
}
|
||||
|
||||
discarded_total = 0
|
||||
for entry in matched_entries:
|
||||
_store_discarded_delivery(
|
||||
dict(entry.get("record") or {}),
|
||||
discarded_by=discarded_by,
|
||||
reason=reason,
|
||||
)
|
||||
try:
|
||||
os.remove(str(entry.get("file_path") or ""))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
discarded_total += 1
|
||||
|
||||
result = {
|
||||
"selector": selector,
|
||||
"limit": limit,
|
||||
"discarded_total": discarded_total,
|
||||
"discarded_by": discarded_by,
|
||||
"reason": reason,
|
||||
"matched_records": _trim_queue_preview(matched_entries),
|
||||
"queue": _delivery_queue_snapshot(),
|
||||
}
|
||||
return True, f"已丢弃 {discarded_total} 条死信记录", result
|
||||
|
||||
|
||||
def _delivery_queue_snapshot() -> dict:
|
||||
pending_dir = _queue_pending_dir()
|
||||
dead_letter_dir = _queue_dead_letter_dir()
|
||||
pending_files = _queue_file_names(pending_dir)
|
||||
dead_letter_files = _queue_file_names(dead_letter_dir)
|
||||
pending_head = _queue_head_record(pending_dir)
|
||||
dead_letter_head = _queue_head_record(dead_letter_dir)
|
||||
pending_count = len(pending_files)
|
||||
dead_letter_count = len(dead_letter_files)
|
||||
|
||||
state = "healthy"
|
||||
label = "正常"
|
||||
reason = "当前没有待重试回执,也没有死信记录。"
|
||||
if dead_letter_count > 0:
|
||||
state = "dead_letter"
|
||||
label = f"死信 {dead_letter_count}"
|
||||
reason = "存在语义失败的回执/事件,自动重试已停止,建议人工查看。"
|
||||
elif pending_count > 0:
|
||||
state = "retrying"
|
||||
label = f"待重试 {pending_count}"
|
||||
reason = "存在待重试的回执/事件,Node Agent 会在后续 heartbeat/poll 周期继续回放。"
|
||||
elif not pending_head and not dead_letter_head:
|
||||
label = "正常"
|
||||
|
||||
return {
|
||||
"state": state,
|
||||
"label": label,
|
||||
"reason": reason,
|
||||
"pending_count": pending_count,
|
||||
"dead_letter_count": dead_letter_count,
|
||||
"oldest_pending_at": str(pending_head.get("created_at") or "").strip(),
|
||||
"oldest_pending_request_id": str(pending_head.get("request_id") or "").strip(),
|
||||
"oldest_pending_kind": str(pending_head.get("kind") or "").strip(),
|
||||
"oldest_dead_letter_at": str(
|
||||
dead_letter_head.get("dead_letter_at") or dead_letter_head.get("created_at") or ""
|
||||
).strip(),
|
||||
"oldest_dead_letter_request_id": str(dead_letter_head.get("request_id") or "").strip(),
|
||||
"oldest_dead_letter_kind": str(dead_letter_head.get("kind") or "").strip(),
|
||||
"last_flush_at": str(_LAST_QUEUE_FLUSH_SUMMARY.get("last_flush_at") or "").strip(),
|
||||
"last_flush_delivered": int(_LAST_QUEUE_FLUSH_SUMMARY.get("delivered") or 0),
|
||||
"last_flush_deferred": int(_LAST_QUEUE_FLUSH_SUMMARY.get("deferred") or 0),
|
||||
"last_flush_dead_letter": int(_LAST_QUEUE_FLUSH_SUMMARY.get("dead_letter") or 0),
|
||||
}
|
||||
|
||||
|
||||
def _response_detail_code(response: dict) -> str:
|
||||
if not isinstance(response, dict):
|
||||
return ""
|
||||
top_level = str(response.get("detail_code") or "").strip()
|
||||
if top_level:
|
||||
return top_level
|
||||
data = response.get("data")
|
||||
if isinstance(data, dict):
|
||||
return str(data.get("detail_code") or "").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _ensure_ok_response(response: dict, fallback_message: str) -> dict:
|
||||
raw_code = response.get("code", 1)
|
||||
try:
|
||||
normalized_code = int(raw_code if raw_code not in (None, "") else 1)
|
||||
except Exception:
|
||||
normalized_code = 1
|
||||
if normalized_code == 0:
|
||||
return response
|
||||
raise AgentResponseError(
|
||||
str(response.get("message") or fallback_message),
|
||||
detail_code=_response_detail_code(response),
|
||||
payload=(response.get("data") if isinstance(response.get("data"), dict) else {}),
|
||||
)
|
||||
|
||||
|
||||
def _build_delivery_record(kind: str, path: str, payload: dict, request_id: str) -> dict:
|
||||
return {
|
||||
"kind": str(kind or "").strip() or "delivery",
|
||||
"path": str(path or "").strip(),
|
||||
"payload": dict(payload or {}),
|
||||
"request_id": str(request_id or "").strip(),
|
||||
"attempt_count": 0,
|
||||
"created_at": _now_text(),
|
||||
"updated_at": _now_text(),
|
||||
"last_error": "",
|
||||
"last_detail_code": "",
|
||||
}
|
||||
|
||||
|
||||
def _store_pending_delivery(record: dict) -> str:
|
||||
_ensure_queue_dirs()
|
||||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||||
file_path = os.path.join(_queue_pending_dir(), file_name)
|
||||
_write_json_file(file_path, record)
|
||||
return file_path
|
||||
|
||||
|
||||
def _store_dead_letter_delivery(record: dict, *, reason: str, detail_code: str = "") -> str:
|
||||
_ensure_queue_dirs()
|
||||
dead_record = dict(record or {})
|
||||
dead_record["dead_letter_reason"] = str(reason or "").strip()
|
||||
dead_record["dead_letter_at"] = _now_text()
|
||||
dead_record["last_error"] = str(reason or "").strip()
|
||||
dead_record["last_detail_code"] = str(detail_code or "").strip()
|
||||
file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json"
|
||||
file_path = os.path.join(_queue_dead_letter_dir(), file_name)
|
||||
_write_json_file(file_path, dead_record)
|
||||
return file_path
|
||||
|
||||
|
||||
def _dispatch_delivery_record(record: dict) -> dict:
|
||||
kind = str(record.get("kind") or "delivery").strip() or "delivery"
|
||||
path = str(record.get("path") or "").strip()
|
||||
payload = record.get("payload") or {}
|
||||
timeout = 15 if kind == "job_event" else 30
|
||||
response = _post(path, payload, timeout=timeout)
|
||||
return _ensure_ok_response(response, f"{kind} failed")
|
||||
|
||||
|
||||
def _flush_delivery_queue(limit: int | None = None) -> dict:
|
||||
global _LAST_QUEUE_FLUSH_SUMMARY
|
||||
_ensure_queue_dirs()
|
||||
safe_limit = max(1, int(limit or AGENT_QUEUE_FLUSH_LIMIT or 1))
|
||||
summary = {
|
||||
"scanned": 0,
|
||||
"delivered": 0,
|
||||
"deferred": 0,
|
||||
"dead_letter": 0,
|
||||
}
|
||||
file_names = sorted(
|
||||
file_name
|
||||
for file_name in os.listdir(_queue_pending_dir())
|
||||
if file_name.endswith(".json")
|
||||
)[:safe_limit]
|
||||
for file_name in file_names:
|
||||
summary["scanned"] += 1
|
||||
file_path = os.path.join(_queue_pending_dir(), file_name)
|
||||
try:
|
||||
record = _read_json_file(file_path)
|
||||
except Exception as exc:
|
||||
_store_dead_letter_delivery({"file_name": file_name}, reason=f"invalid queue record: {exc}")
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
summary["dead_letter"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
_dispatch_delivery_record(record)
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
summary["delivered"] += 1
|
||||
except AgentResponseError as exc:
|
||||
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
|
||||
record["updated_at"] = _now_text()
|
||||
record["last_error"] = str(exc)
|
||||
record["last_detail_code"] = exc.detail_code
|
||||
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
|
||||
_store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
summary["dead_letter"] += 1
|
||||
_log(
|
||||
f"delivery moved to dead-letter: kind={record.get('kind')} request_id={record.get('request_id')} "
|
||||
f"detail_code={exc.detail_code or '-'} message={exc}"
|
||||
)
|
||||
else:
|
||||
_write_json_file(file_path, record)
|
||||
summary["deferred"] += 1
|
||||
except Exception as exc:
|
||||
record["attempt_count"] = int(record.get("attempt_count") or 0) + 1
|
||||
record["updated_at"] = _now_text()
|
||||
record["last_error"] = str(exc)
|
||||
_write_json_file(file_path, record)
|
||||
summary["deferred"] += 1
|
||||
summary["last_flush_at"] = _now_text()
|
||||
_LAST_QUEUE_FLUSH_SUMMARY = dict(summary)
|
||||
return summary
|
||||
|
||||
|
||||
def _deliver_or_queue(
|
||||
*,
|
||||
kind: str,
|
||||
path: str,
|
||||
payload: dict,
|
||||
request_id: str,
|
||||
timeout: int = 30,
|
||||
) -> dict:
|
||||
record = _build_delivery_record(kind, path, payload, request_id)
|
||||
try:
|
||||
response = _post(path, payload, timeout=timeout)
|
||||
_ensure_ok_response(response, f"{kind} failed")
|
||||
return {"state": "delivered", "request_id": request_id, "detail_code": "", "response": response}
|
||||
except AgentResponseError as exc:
|
||||
if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES:
|
||||
file_path = _store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code)
|
||||
_log(
|
||||
f"{kind} dead-lettered: request_id={request_id} detail_code={exc.detail_code or '-'} "
|
||||
f"message={exc} file={file_path}"
|
||||
)
|
||||
return {"state": "dead_letter", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
|
||||
file_path = _store_pending_delivery({**record, "last_error": str(exc), "last_detail_code": exc.detail_code})
|
||||
_log(
|
||||
f"{kind} queued for retry: request_id={request_id} detail_code={exc.detail_code or '-'} "
|
||||
f"message={exc} file={file_path}"
|
||||
)
|
||||
return {"state": "queued", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path}
|
||||
except Exception as exc:
|
||||
file_path = _store_pending_delivery({**record, "last_error": str(exc)})
|
||||
_log(f"{kind} queued for retry: request_id={request_id} message={exc} file={file_path}")
|
||||
return {"state": "queued", "request_id": request_id, "detail_code": "", "file_path": file_path}
|
||||
|
||||
|
||||
def _headers() -> dict[str, str]:
|
||||
return {
|
||||
"Content-Type": "application/json",
|
||||
"X-Domaincheck-Agent-Token": AGENT_TOKEN,
|
||||
}
|
||||
|
||||
|
||||
def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict:
|
||||
if not CONTROL_PLANE_BASE_URL:
|
||||
raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置")
|
||||
if not AGENT_TOKEN:
|
||||
raise RuntimeError("OPS_AGENT_TOKEN 未配置")
|
||||
url = f"{CONTROL_PLANE_BASE_URL}{path}"
|
||||
data = json.dumps(payload or {}, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(url=url, data=data, headers=_headers(), method=method.upper())
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read().decode("utf-8", errors="ignore")
|
||||
return json.loads(body or "{}")
|
||||
|
||||
|
||||
def _post(path: str, payload: dict, timeout: int = 30) -> dict:
|
||||
return _request("POST", path, payload, timeout=timeout)
|
||||
|
||||
|
||||
def _hostname() -> str:
|
||||
try:
|
||||
return socket.gethostname()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _ip() -> str:
|
||||
try:
|
||||
return socket.gethostbyname(socket.gethostname())
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _base_payload() -> dict:
|
||||
return {
|
||||
"node_code": NODE_CODE,
|
||||
"region": NODE_REGION,
|
||||
"role": NODE_ROLE,
|
||||
"title": NODE_CODE,
|
||||
"hostname": _hostname(),
|
||||
"ip": _ip(),
|
||||
"agent_version": AGENT_VERSION,
|
||||
"capabilities": AGENT_CAPABILITIES,
|
||||
"labels": AGENT_LABELS,
|
||||
"metadata": {
|
||||
"service_names": {
|
||||
"api": API_SERVICE_NAME,
|
||||
"worker": WORKER_SERVICE_NAME,
|
||||
"sync_agent": SYNC_AGENT_SERVICE_NAME,
|
||||
"node_agent": NODE_AGENT_SERVICE_NAME,
|
||||
},
|
||||
"delivery_queue": _delivery_queue_snapshot(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]:
|
||||
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
return completed.returncode, completed.stdout.strip(), completed.stderr.strip()
|
||||
|
||||
|
||||
def _execute_action(
|
||||
action: str,
|
||||
payload: dict,
|
||||
*,
|
||||
job_id: int | None = None,
|
||||
job_context: dict | None = None,
|
||||
) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action == "delivery.queue.flush":
|
||||
limit = _normalize_queue_limit((payload or {}).get("limit"), default=20)
|
||||
summary = _flush_delivery_queue(limit=limit)
|
||||
return True, "Delivery Queue 已执行冲刷", {
|
||||
"action": normalized_action,
|
||||
"limit": limit,
|
||||
"flush_summary": summary,
|
||||
"queue": _delivery_queue_snapshot(),
|
||||
}
|
||||
if normalized_action == "delivery.queue.replay":
|
||||
return _replay_dead_letter_records(payload)
|
||||
if normalized_action == "delivery.queue.discard":
|
||||
return _discard_dead_letter_records(payload)
|
||||
if supports_structured_action(normalized_action):
|
||||
return execute_structured_action(
|
||||
normalized_action,
|
||||
payload,
|
||||
service_names=build_service_name_map(
|
||||
api_service_name=API_SERVICE_NAME,
|
||||
worker_service_name=WORKER_SERVICE_NAME,
|
||||
sync_agent_service_name=SYNC_AGENT_SERVICE_NAME,
|
||||
node_agent_service_name=NODE_AGENT_SERVICE_NAME,
|
||||
),
|
||||
runner=_run,
|
||||
host_context={
|
||||
"hostname": _hostname(),
|
||||
"ip": _ip(),
|
||||
},
|
||||
)
|
||||
if normalized_action == "deploy.release":
|
||||
return execute_release_action(
|
||||
dict(payload or {}),
|
||||
run_command=_run,
|
||||
default_api_service_name=API_SERVICE_NAME,
|
||||
event_callback=(
|
||||
(lambda event_type, message, level="info", payload=None: _job_event(
|
||||
int(job_id),
|
||||
event_type=event_type,
|
||||
message=message,
|
||||
level=level,
|
||||
payload={
|
||||
**dict(payload or {}),
|
||||
"release_context": dict((job_context or {}).get("release_context") or {}),
|
||||
"step_key": str((job_context or {}).get("step_key") or "").strip(),
|
||||
},
|
||||
summary_text=message,
|
||||
focus_ref=dict((job_context or {}).get("focus_ref") or {}),
|
||||
occurred_at=_now_text(),
|
||||
))
|
||||
if job_id
|
||||
else None
|
||||
),
|
||||
urlopen_func=urllib.request.urlopen,
|
||||
user_agent=f"domaincheck-node-agent/{AGENT_VERSION}",
|
||||
)
|
||||
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}
|
||||
|
||||
|
||||
def _register() -> None:
|
||||
response = _post("/api/v1/ops/agent/register", _base_payload())
|
||||
_ensure_ok_response(response, "agent register failed")
|
||||
_log(f"registered: {response.get('message')}")
|
||||
|
||||
|
||||
def _heartbeat() -> None:
|
||||
response = _post("/api/v1/ops/agent/heartbeat", _base_payload())
|
||||
_ensure_ok_response(response, "agent heartbeat failed")
|
||||
|
||||
|
||||
def _pull_jobs() -> list[dict]:
|
||||
response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE})
|
||||
_ensure_ok_response(response, "agent pull failed")
|
||||
data = response.get("data") or {}
|
||||
return list(data.get("jobs") or [])
|
||||
|
||||
|
||||
def _normalize_agent_job(job: dict) -> dict:
|
||||
normalized_job = dict(job or {})
|
||||
focus_ref = normalized_job.get("focus_ref") if isinstance(normalized_job.get("focus_ref"), dict) else {}
|
||||
release_context = (
|
||||
normalized_job.get("release_context") if isinstance(normalized_job.get("release_context"), dict) else {}
|
||||
)
|
||||
step_ref = normalized_job.get("step_ref") if isinstance(normalized_job.get("step_ref"), dict) else {}
|
||||
job_id = int(normalized_job.get("job_id") or normalized_job.get("id") or 0)
|
||||
step_key = str(
|
||||
normalized_job.get("step_key")
|
||||
or step_ref.get("step_key")
|
||||
or "dispatch"
|
||||
).strip() or "dispatch"
|
||||
step_title = str(
|
||||
normalized_job.get("step_title")
|
||||
or step_ref.get("step_title")
|
||||
or normalized_job.get("summary")
|
||||
or normalized_job.get("action")
|
||||
or step_key
|
||||
).strip() or step_key
|
||||
if not focus_ref:
|
||||
focus_ref = {
|
||||
"kind": "ops_job",
|
||||
"job_id": job_id,
|
||||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||||
"action": str(normalized_job.get("action") or "").strip(),
|
||||
"target_node_code": str(normalized_job.get("target_node_code") or "").strip(),
|
||||
}
|
||||
return {
|
||||
**normalized_job,
|
||||
"job_id": job_id,
|
||||
"job_code": str(normalized_job.get("job_code") or "").strip(),
|
||||
"job_type": str(normalized_job.get("job_type") or "ops_action").strip() or "ops_action",
|
||||
"action": str(normalized_job.get("action") or "").strip(),
|
||||
"payload": dict(normalized_job.get("payload") or {}),
|
||||
"policy": dict(normalized_job.get("policy") or {}),
|
||||
"focus_ref": focus_ref,
|
||||
"release_context": dict(release_context or {}),
|
||||
"step_key": step_key,
|
||||
"step_title": step_title,
|
||||
"step_ref": {
|
||||
"step_id": int(step_ref.get("step_id") or 0),
|
||||
"step_key": step_key,
|
||||
"step_title": step_title,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _job_start(job_id: int, *, job: dict | None = None) -> None:
|
||||
normalized_job = _normalize_agent_job(job or {"job_id": job_id})
|
||||
response = _post(
|
||||
f"/api/v1/ops/agent/jobs/{job_id}/start",
|
||||
{
|
||||
"node_code": NODE_CODE,
|
||||
"job_code": normalized_job.get("job_code") or "",
|
||||
"action": normalized_job.get("action") or "",
|
||||
"step_key": normalized_job.get("step_key") or "",
|
||||
"focus_ref": normalized_job.get("focus_ref") or {},
|
||||
},
|
||||
)
|
||||
_ensure_ok_response(response, "job start failed")
|
||||
|
||||
|
||||
def _job_complete(
|
||||
job_id: int,
|
||||
*,
|
||||
status: str,
|
||||
stdout: str,
|
||||
stderr: str,
|
||||
result: dict,
|
||||
error_message: str = "",
|
||||
client_request_id: str | None = None,
|
||||
duration_ms: int | None = None,
|
||||
summary_text: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
step_ref: dict | None = None,
|
||||
release_context: dict | None = None,
|
||||
) -> dict:
|
||||
request_id = str(client_request_id or "").strip() or _new_request_id("complete", job_id)
|
||||
payload = {
|
||||
"node_code": NODE_CODE,
|
||||
"status": status,
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
"result": result,
|
||||
"error_message": error_message,
|
||||
"client_request_id": request_id,
|
||||
}
|
||||
if duration_ms is not None:
|
||||
payload["duration_ms"] = max(0, int(duration_ms or 0))
|
||||
if str(summary_text or "").strip():
|
||||
payload["summary_text"] = str(summary_text).strip()
|
||||
if isinstance(focus_ref, dict) and focus_ref:
|
||||
payload["focus_ref"] = dict(focus_ref)
|
||||
if isinstance(step_ref, dict) and step_ref:
|
||||
payload["step_ref"] = dict(step_ref)
|
||||
if isinstance(release_context, dict) and release_context:
|
||||
payload["release_context"] = dict(release_context)
|
||||
return _deliver_or_queue(
|
||||
kind="job_complete",
|
||||
path=f"/api/v1/ops/agent/jobs/{job_id}/complete",
|
||||
payload=payload,
|
||||
request_id=request_id,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def _job_event(
|
||||
job_id: int,
|
||||
*,
|
||||
event_type: str,
|
||||
message: str,
|
||||
level: str = "info",
|
||||
payload: dict | None = None,
|
||||
client_event_id: str | None = None,
|
||||
summary_text: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
occurred_at: str = "",
|
||||
) -> dict:
|
||||
request_id = str(client_event_id or "").strip() or _new_request_id("event", job_id)
|
||||
delivery_payload = {
|
||||
"node_code": NODE_CODE,
|
||||
"event_type": event_type,
|
||||
"message": message,
|
||||
"level": level,
|
||||
"payload": payload or {},
|
||||
"client_event_id": request_id,
|
||||
}
|
||||
if str(summary_text or "").strip():
|
||||
delivery_payload["summary_text"] = str(summary_text).strip()
|
||||
if isinstance(focus_ref, dict) and focus_ref:
|
||||
delivery_payload["focus_ref"] = dict(focus_ref)
|
||||
if str(occurred_at or "").strip():
|
||||
delivery_payload["occurred_at"] = str(occurred_at).strip()
|
||||
return _deliver_or_queue(
|
||||
kind="job_event",
|
||||
path=f"/api/v1/ops/agent/jobs/{job_id}/events",
|
||||
payload=delivery_payload,
|
||||
request_id=request_id,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
def _process_job(job: dict) -> None:
|
||||
normalized_job = _normalize_agent_job(job)
|
||||
job_id = int(normalized_job.get("job_id") or 0)
|
||||
action = str(normalized_job.get("action") or "").strip()
|
||||
payload = dict(normalized_job.get("payload") or {})
|
||||
if job_id <= 0 or not action:
|
||||
return
|
||||
|
||||
started_at = time.monotonic()
|
||||
_log(
|
||||
"job start: "
|
||||
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
|
||||
f"type={normalized_job.get('job_type') or '-'} "
|
||||
f"step={normalized_job.get('step_key') or '-'} action={action}"
|
||||
)
|
||||
start_delivery_state = "delivered"
|
||||
start_delivery_error = ""
|
||||
try:
|
||||
_job_start(job_id, job=normalized_job)
|
||||
except Exception as exc:
|
||||
start_delivery_state = "failed_local"
|
||||
start_delivery_error = str(exc)
|
||||
_log(
|
||||
"job start delivery failed, continue locally: "
|
||||
f"id={job_id} code={normalized_job.get('job_code') or '-'} action={action} error={exc}"
|
||||
)
|
||||
_job_event(
|
||||
job_id,
|
||||
event_type="executor_received",
|
||||
message=f"node agent accepted action {action}",
|
||||
summary_text=f"已接单 {action}",
|
||||
focus_ref=dict(normalized_job.get("focus_ref") or {}),
|
||||
occurred_at=_now_text(),
|
||||
payload={
|
||||
"action": action,
|
||||
"job_code": normalized_job.get("job_code") or "",
|
||||
"job_type": normalized_job.get("job_type") or "",
|
||||
"step_key": normalized_job.get("step_key") or "",
|
||||
"step_title": normalized_job.get("step_title") or "",
|
||||
"release_context": dict(normalized_job.get("release_context") or {}),
|
||||
"start_delivery_state": start_delivery_state,
|
||||
"start_delivery_error": start_delivery_error,
|
||||
},
|
||||
)
|
||||
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
|
||||
stdout = str(result.get("stdout") or "")
|
||||
stderr = str(result.get("stderr") or "")
|
||||
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
|
||||
summary_text = str(result.get("summary_text") or result.get("summary") or message).strip()
|
||||
delivery = _job_complete(
|
||||
job_id,
|
||||
status="success" if ok else "failed",
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
result=result,
|
||||
error_message="" if ok else message,
|
||||
duration_ms=duration_ms,
|
||||
summary_text=summary_text,
|
||||
focus_ref=dict(normalized_job.get("focus_ref") or {}),
|
||||
step_ref=dict(normalized_job.get("step_ref") or {}),
|
||||
release_context=dict(normalized_job.get("release_context") or {}),
|
||||
)
|
||||
_log(
|
||||
"job complete: "
|
||||
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
|
||||
f"action={action} ok={ok} duration_ms={duration_ms} delivery={delivery.get('state')}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not NODE_CODE:
|
||||
raise RuntimeError("NODE_CODE 未配置")
|
||||
_log(f"starting node agent: node={NODE_CODE} role={NODE_ROLE} region={NODE_REGION}")
|
||||
_ensure_queue_dirs()
|
||||
_register()
|
||||
last_heartbeat_at = 0.0
|
||||
|
||||
while True:
|
||||
now = time.time()
|
||||
try:
|
||||
delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT)
|
||||
if delivery_summary["delivered"] or delivery_summary["dead_letter"]:
|
||||
_log(f"delivery queue flush: {delivery_summary}")
|
||||
if now - last_heartbeat_at >= 15:
|
||||
_heartbeat()
|
||||
last_heartbeat_at = now
|
||||
jobs = _pull_jobs()
|
||||
if jobs:
|
||||
for job in jobs:
|
||||
_process_job(job)
|
||||
else:
|
||||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||||
except urllib.error.HTTPError as exc:
|
||||
_log(f"http error: {exc.code}")
|
||||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||||
except urllib.error.URLError as exc:
|
||||
_log(f"url error: {exc}")
|
||||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||||
except Exception as exc:
|
||||
_log(f"loop error: {exc}")
|
||||
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,8 +1,10 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ApiResponse(BaseModel):
|
||||
code: int = 0
|
||||
message: str = "ok"
|
||||
data: Any = None
|
||||
detail_code: Optional[str] = None
|
||||
|
||||
241
domain-api/app/services/build_info_service.py
Normal file
241
domain-api/app/services/build_info_service.py
Normal file
@@ -0,0 +1,241 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
DOMAIN_API_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKSPACE_ROOT = DOMAIN_API_ROOT.parent
|
||||
_REGISTERED_ROUTE_PATHS: set[str] = set()
|
||||
|
||||
|
||||
def remember_registered_route_paths(route_paths: Iterable[str] | None) -> None:
|
||||
global _REGISTERED_ROUTE_PATHS
|
||||
normalized = {
|
||||
str(item).strip()
|
||||
for item in list(route_paths or [])
|
||||
if str(item).strip()
|
||||
}
|
||||
_REGISTERED_ROUTE_PATHS = normalized
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _discover_manifest_candidate() -> tuple[Path | None, str]:
|
||||
configured = str(settings.build_manifest_path or "").strip()
|
||||
if configured:
|
||||
path = Path(configured)
|
||||
if path.exists():
|
||||
return path, "configured_manifest"
|
||||
|
||||
root_manifest = WORKSPACE_ROOT / "release_manifest.json"
|
||||
if root_manifest.exists():
|
||||
return root_manifest, "release_manifest"
|
||||
|
||||
latest_release = WORKSPACE_ROOT / "release" / "latest_release.json"
|
||||
if latest_release.exists():
|
||||
return latest_release, "latest_release"
|
||||
|
||||
return None, ""
|
||||
|
||||
|
||||
def _build_info_from_env() -> dict | None:
|
||||
if not any(
|
||||
[
|
||||
str(settings.build_commit_sha or "").strip(),
|
||||
str(settings.build_commit_ref or "").strip(),
|
||||
str(settings.build_generated_at or "").strip(),
|
||||
str(settings.build_package_name or "").strip(),
|
||||
str(settings.build_checksum or "").strip(),
|
||||
]
|
||||
):
|
||||
return None
|
||||
|
||||
return {
|
||||
"source": str(settings.build_source_label or "").strip() or "env",
|
||||
"package_name": str(settings.build_package_name or "").strip(),
|
||||
"generated_at": str(settings.build_generated_at or "").strip(),
|
||||
"commit_sha": str(settings.build_commit_sha or "").strip(),
|
||||
"commit_ref": str(settings.build_commit_ref or "").strip(),
|
||||
"checksum": str(settings.build_checksum or "").strip(),
|
||||
"manifest_path": str(settings.build_manifest_path or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _build_info_from_manifest() -> dict | None:
|
||||
manifest_path, source = _discover_manifest_candidate()
|
||||
if manifest_path is None:
|
||||
return None
|
||||
|
||||
payload = _read_json(manifest_path)
|
||||
if not payload:
|
||||
return {
|
||||
"source": source,
|
||||
"package_name": "",
|
||||
"generated_at": "",
|
||||
"commit_sha": "",
|
||||
"commit_ref": "",
|
||||
"checksum": "",
|
||||
"manifest_path": str(manifest_path),
|
||||
}
|
||||
|
||||
checksum = str(payload.get("checksum") or payload.get("sha256") or "").strip()
|
||||
return {
|
||||
"source": source,
|
||||
"package_name": str(payload.get("package_name") or "").strip(),
|
||||
"generated_at": str(payload.get("generated_at") or "").strip(),
|
||||
"commit_sha": str(payload.get("commit_sha") or "").strip(),
|
||||
"commit_ref": str(payload.get("commit_ref") or "").strip(),
|
||||
"checksum": checksum,
|
||||
"manifest_path": str(manifest_path),
|
||||
}
|
||||
|
||||
|
||||
def _build_info_from_git() -> dict | None:
|
||||
try:
|
||||
inside_worktree = subprocess.run(
|
||||
["git", "-C", str(WORKSPACE_ROOT), "rev-parse", "--is-inside-work-tree"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if inside_worktree.returncode != 0 or str(inside_worktree.stdout or "").strip() != "true":
|
||||
return None
|
||||
|
||||
def _read_git(*args: str) -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "-C", str(WORKSPACE_ROOT), *args],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
)
|
||||
except Exception:
|
||||
return ""
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
return str(result.stdout or "").strip()
|
||||
|
||||
return {
|
||||
"source": "git",
|
||||
"package_name": "",
|
||||
"generated_at": "",
|
||||
"commit_sha": _read_git("rev-parse", "--short=12", "HEAD"),
|
||||
"commit_ref": _read_git("rev-parse", "--abbrev-ref", "HEAD"),
|
||||
"checksum": "",
|
||||
"manifest_path": "",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_build_identity() -> dict:
|
||||
return (
|
||||
_build_info_from_env()
|
||||
or _build_info_from_manifest()
|
||||
or _build_info_from_git()
|
||||
or {
|
||||
"source": "unknown",
|
||||
"package_name": "",
|
||||
"generated_at": "",
|
||||
"commit_sha": "",
|
||||
"commit_ref": "",
|
||||
"checksum": "",
|
||||
"manifest_path": "",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _expected_route_paths() -> dict[str, str]:
|
||||
prefix = str(settings.api_prefix or "/api/v1").rstrip("/")
|
||||
return {
|
||||
"ops_contracts": f"{prefix}/ops/contracts",
|
||||
"ops_contract_detail": f"{prefix}/ops/contracts/{{contract_key}}",
|
||||
"ops_stack_diagnosis": f"{prefix}/ops/stack-diagnosis",
|
||||
"ops_node_handover": f"{prefix}/ops/nodes/{{node_code}}/handover",
|
||||
"ops_node_onboarding": f"{prefix}/ops/nodes/{{node_code}}/onboarding",
|
||||
"ops_node_onboarding_bootstrap_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/preview",
|
||||
"ops_node_onboarding_bootstrap_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/execute",
|
||||
"ops_node_onboarding_acceptance_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/preview",
|
||||
"ops_node_onboarding_acceptance_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/execute",
|
||||
"ops_node_onboarding_recovery_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/preview",
|
||||
"ops_node_onboarding_recovery_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/execute",
|
||||
"ops_node_scene_log": f"{prefix}/ops/nodes/{{node_code}}/scene-log",
|
||||
"ops_node_handover_bootstrap_plan": f"{prefix}/ops/nodes/{{node_code}}/handover/bootstrap-plan",
|
||||
"ops_driver_feed": f"{prefix}/ops/driver-feed",
|
||||
"ops_codex_brief": f"{prefix}/ops/codex-brief",
|
||||
"ops_activity_stream": f"{prefix}/ops/activity-stream",
|
||||
"ops_runbook_resolve": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/resolve",
|
||||
"ops_runbook_execute": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/execute",
|
||||
"runtime_build_info": f"{prefix}/runtime/build-info",
|
||||
}
|
||||
|
||||
|
||||
def _read_text(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _repository_capabilities() -> dict:
|
||||
ops_agent_service_text = _read_text(DOMAIN_API_ROOT / "app" / "services" / "ops_agent_service.py")
|
||||
bootstrap_node_agent_text = _read_text(DOMAIN_API_ROOT / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
|
||||
return {
|
||||
"supports_install_command_block": (
|
||||
"install_command_block" in ops_agent_service_text
|
||||
and "_candidate_node_agent_install_paths" in ops_agent_service_text
|
||||
),
|
||||
"supports_multi_layout_bootstrap": "resolve_project_root" in bootstrap_node_agent_text,
|
||||
}
|
||||
|
||||
|
||||
def _build_route_surface(route_paths: Iterable[str] | None = None) -> dict:
|
||||
expected = _expected_route_paths()
|
||||
normalized_paths = {
|
||||
str(item).strip()
|
||||
for item in (list(route_paths) if route_paths is not None else list(_REGISTERED_ROUTE_PATHS))
|
||||
if str(item).strip()
|
||||
}
|
||||
mode = "registered" if normalized_paths else "declared_contract"
|
||||
flags = {key: (path in normalized_paths if normalized_paths else True) for key, path in expected.items()}
|
||||
missing_keys = [key for key, available in flags.items() if not available]
|
||||
return {
|
||||
"mode": mode,
|
||||
"registered_paths_total": len(normalized_paths),
|
||||
"surface_flags": flags,
|
||||
"expected_paths": expected,
|
||||
"missing_keys": missing_keys,
|
||||
"missing_paths": [expected[key] for key in missing_keys],
|
||||
"surface_complete": len(missing_keys) == 0,
|
||||
}
|
||||
|
||||
|
||||
def get_runtime_build_info(route_paths: Iterable[str] | None = None) -> dict:
|
||||
build = _resolve_build_identity()
|
||||
route_surface = _build_route_surface(route_paths=route_paths)
|
||||
return {
|
||||
"source": str(build.get("source") or "").strip() or "unknown",
|
||||
"package_name": str(build.get("package_name") or "").strip(),
|
||||
"generated_at": str(build.get("generated_at") or "").strip(),
|
||||
"commit_sha": str(build.get("commit_sha") or "").strip(),
|
||||
"commit_ref": str(build.get("commit_ref") or "").strip(),
|
||||
"checksum": str(build.get("checksum") or "").strip(),
|
||||
"manifest_path": str(build.get("manifest_path") or "").strip(),
|
||||
"workspace_root": str(WORKSPACE_ROOT),
|
||||
"domain_api_root": str(DOMAIN_API_ROOT),
|
||||
"repository_capabilities": _repository_capabilities(),
|
||||
"route_surface": route_surface,
|
||||
}
|
||||
@@ -147,15 +147,50 @@ def _build_remote_log_lines(
|
||||
mode: str,
|
||||
limit: int = 240,
|
||||
) -> list[str]:
|
||||
return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)["lines"]
|
||||
|
||||
|
||||
def _build_remote_log_snapshot(
|
||||
active_job: dict | None,
|
||||
*,
|
||||
enabled: bool,
|
||||
mode: str,
|
||||
limit: int = 240,
|
||||
) -> dict:
|
||||
if not enabled:
|
||||
return []
|
||||
return {
|
||||
"lines": [],
|
||||
"line_count": 0,
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
"source_nodes": [],
|
||||
"source_node_count": 0,
|
||||
}
|
||||
if not active_job:
|
||||
return []
|
||||
return {
|
||||
"lines": [],
|
||||
"line_count": 0,
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
"source_nodes": [],
|
||||
"source_node_count": 0,
|
||||
}
|
||||
events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
|
||||
if not events:
|
||||
return []
|
||||
return {
|
||||
"lines": [],
|
||||
"line_count": 0,
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
"source_nodes": [],
|
||||
"source_node_count": 0,
|
||||
}
|
||||
|
||||
lines: list[str] = []
|
||||
source_nodes: set[str] = set()
|
||||
source_node_summaries: dict[str, dict] = {}
|
||||
last_at = ""
|
||||
last_line = ""
|
||||
normalized_mode = str(mode or "key").strip().lower()
|
||||
if normalized_mode not in {"key", "full"}:
|
||||
normalized_mode = "key"
|
||||
@@ -180,8 +215,47 @@ def _build_remote_log_lines(
|
||||
continue
|
||||
if len(message) > _REMOTE_LOG_MAX_CHARS:
|
||||
message = f"{message[:_REMOTE_LOG_MAX_CHARS]}..."
|
||||
lines.append(f"[{created_at}] [{node_code}] {message}")
|
||||
return lines[-max(1, int(limit or 240)) :]
|
||||
formatted_line = f"[{created_at}] [{node_code}] {message}"
|
||||
lines.append(formatted_line)
|
||||
source_nodes.add(node_code)
|
||||
node_summary = source_node_summaries.setdefault(
|
||||
node_code,
|
||||
{
|
||||
"node_code": node_code,
|
||||
"line_count": 0,
|
||||
"key_line_count": 0,
|
||||
"full_line_count": 0,
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
},
|
||||
)
|
||||
node_summary["line_count"] += 1
|
||||
if event_mode == "full":
|
||||
node_summary["full_line_count"] += 1
|
||||
else:
|
||||
node_summary["key_line_count"] += 1
|
||||
node_summary["last_at"] = created_at
|
||||
node_summary["last_line"] = formatted_line
|
||||
last_at = created_at
|
||||
last_line = formatted_line
|
||||
sliced_lines = lines[-max(1, int(limit or 240)) :]
|
||||
sorted_source_node_summaries = sorted(
|
||||
source_node_summaries.values(),
|
||||
key=lambda item: (
|
||||
str(item.get("last_at") or ""),
|
||||
str(item.get("node_code") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return {
|
||||
"lines": sliced_lines,
|
||||
"line_count": len(sliced_lines),
|
||||
"last_at": last_at,
|
||||
"last_line": last_line,
|
||||
"source_nodes": sorted(source_nodes),
|
||||
"source_node_count": len(source_nodes),
|
||||
"source_node_summaries": sorted_source_node_summaries,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_remote_log_lines(
|
||||
@@ -195,6 +269,17 @@ def _resolve_remote_log_lines(
|
||||
return _build_remote_log_lines(active_job, enabled=enabled, mode=mode, limit=limit)
|
||||
|
||||
|
||||
def _resolve_remote_log_snapshot(
|
||||
active_job: dict | None,
|
||||
runs: list[dict],
|
||||
*,
|
||||
enabled: bool,
|
||||
mode: str,
|
||||
limit: int = 240,
|
||||
) -> dict:
|
||||
return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)
|
||||
|
||||
|
||||
def _load_runtime_state() -> dict:
|
||||
try:
|
||||
redis_client = get_redis()
|
||||
@@ -404,13 +489,14 @@ def get_detect_status() -> dict:
|
||||
runs = sync_detect_runs(runtime_snapshot, progress, settings_summary, active_job=active_job)
|
||||
worker_log_sync_enabled = bool(runtime_settings.get("worker_log_sync_enabled", False))
|
||||
worker_log_sync_mode = str(runtime_settings.get("worker_log_sync_mode", "key") or "key")
|
||||
remote_log_lines = _resolve_remote_log_lines(
|
||||
remote_log_snapshot = _resolve_remote_log_snapshot(
|
||||
active_job,
|
||||
runs,
|
||||
enabled=worker_log_sync_enabled,
|
||||
mode=worker_log_sync_mode,
|
||||
limit=240,
|
||||
)
|
||||
remote_log_lines = list(remote_log_snapshot.get("lines") or [])
|
||||
dependency_alerts = _extract_dependency_alerts(recent_lines)
|
||||
append_detect_result_projection_if_changed(
|
||||
detect={
|
||||
@@ -465,6 +551,12 @@ def get_detect_status() -> dict:
|
||||
"recent_warning": recent_proxy_warning,
|
||||
"log_lines": recent_lines,
|
||||
"remote_log_lines": remote_log_lines,
|
||||
"remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0),
|
||||
"remote_log_last_at": str(remote_log_snapshot.get("last_at") or ""),
|
||||
"remote_log_last_line": str(remote_log_snapshot.get("last_line") or ""),
|
||||
"remote_log_nodes": list(remote_log_snapshot.get("source_nodes") or []),
|
||||
"remote_log_node_count": int(remote_log_snapshot.get("source_node_count", 0) or 0),
|
||||
"remote_log_node_summaries": list(remote_log_snapshot.get("source_node_summaries") or []),
|
||||
"runs": runs,
|
||||
"active_job": active_job,
|
||||
"worker_log_sync_enabled": worker_log_sync_enabled,
|
||||
|
||||
278
domain-api/app/services/ops_action_executor_core.py
Normal file
278
domain-api/app/services/ops_action_executor_core.py
Normal file
@@ -0,0 +1,278 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
STRUCTURED_ACTIONS = {
|
||||
"health.snapshot",
|
||||
"service.status",
|
||||
"service.restart",
|
||||
"service.start",
|
||||
"service.stop",
|
||||
"logs.collect",
|
||||
"diagnostics.collect",
|
||||
"runtime.start_worker",
|
||||
"runtime.stop_worker",
|
||||
"runtime.restart_api",
|
||||
"runtime.start_sync_agent",
|
||||
"runtime.stop_sync_agent",
|
||||
}
|
||||
|
||||
|
||||
class CommandRunner(Protocol):
|
||||
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
|
||||
...
|
||||
|
||||
|
||||
def supports_structured_action(action: str) -> bool:
|
||||
return str(action or "").strip() in STRUCTURED_ACTIONS
|
||||
|
||||
|
||||
def trim_output(text: str, limit: int) -> str:
|
||||
normalized = str(text or "").strip()
|
||||
if len(normalized) <= int(limit):
|
||||
return normalized
|
||||
return normalized[-int(limit):]
|
||||
|
||||
|
||||
def _read_tail_lines(path: Path, *, max_lines: int) -> list[str]:
|
||||
if not path.exists() or not path.is_file():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read().splitlines()[-max_lines:]
|
||||
|
||||
|
||||
def _runtime_logs_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / "runtime" / "logs"
|
||||
|
||||
|
||||
def _service_log_file_candidates(service_name: str, *, service_names: dict[str, str]) -> list[Path]:
|
||||
normalized_service_name = str(service_name or "").strip()
|
||||
if not normalized_service_name:
|
||||
return []
|
||||
|
||||
domain_root = Path(settings.domain_root)
|
||||
runtime_logs_dir = _runtime_logs_dir()
|
||||
candidates: list[Path] = []
|
||||
|
||||
def append_candidate(path: Path) -> None:
|
||||
if path not in candidates:
|
||||
candidates.append(path)
|
||||
|
||||
if normalized_service_name == (service_names.get("worker") or "domaincheck-worker"):
|
||||
append_candidate(domain_root / "detect_worker.log")
|
||||
append_candidate(domain_root / "logs" / "detect_worker.log")
|
||||
elif normalized_service_name == (service_names.get("api") or "domaincheck-api"):
|
||||
append_candidate(runtime_logs_dir / "domain-api.stderr.log")
|
||||
append_candidate(runtime_logs_dir / "domain-api.stdout.log")
|
||||
append_candidate(domain_root / "logs" / "app.log")
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
def _collect_log_file_fallback(
|
||||
service_name: str,
|
||||
*,
|
||||
lines: int,
|
||||
service_names: dict[str, str],
|
||||
) -> tuple[str, list[str]]:
|
||||
aggregated_lines: list[str] = []
|
||||
used_paths: list[str] = []
|
||||
for path in _service_log_file_candidates(service_name, service_names=service_names):
|
||||
current_lines = _read_tail_lines(path, max_lines=lines)
|
||||
if not current_lines:
|
||||
continue
|
||||
aggregated_lines.extend(current_lines)
|
||||
used_paths.append(str(path))
|
||||
if not aggregated_lines:
|
||||
return "", []
|
||||
return "\n".join(aggregated_lines[-lines:]), used_paths
|
||||
|
||||
|
||||
def build_service_name_map(
|
||||
*,
|
||||
api_service_name: str,
|
||||
worker_service_name: str,
|
||||
sync_agent_service_name: str,
|
||||
node_agent_service_name: str = "domaincheck-node-agent",
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"api": str(api_service_name or "").strip() or "domaincheck-api",
|
||||
"worker": str(worker_service_name or "").strip() or "domaincheck-worker",
|
||||
"sync_agent": str(sync_agent_service_name or "").strip() or "domaincheck-sync-agent",
|
||||
"node_agent": str(node_agent_service_name or "").strip() or "domaincheck-node-agent",
|
||||
}
|
||||
|
||||
|
||||
def service_name_from_payload(payload: dict, *, default_worker_service_name: str) -> str:
|
||||
service_name = str((payload or {}).get("service_name") or "").strip()
|
||||
return service_name or str(default_worker_service_name or "").strip() or "domaincheck-worker"
|
||||
|
||||
|
||||
def service_name_for_action(action: str, payload: dict, service_names: dict[str, str]) -> str:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action in {"service.status", "service.restart", "service.start", "service.stop"}:
|
||||
return service_name_from_payload(payload, default_worker_service_name=service_names.get("worker") or "")
|
||||
runtime_action_map = {
|
||||
"runtime.start_worker": service_names.get("worker") or "domaincheck-worker",
|
||||
"runtime.stop_worker": service_names.get("worker") or "domaincheck-worker",
|
||||
"runtime.restart_api": service_names.get("api") or "domaincheck-api",
|
||||
"runtime.start_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
|
||||
"runtime.stop_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
|
||||
}
|
||||
return str(runtime_action_map.get(normalized_action) or "").strip()
|
||||
|
||||
|
||||
def systemctl_action_name(action: str) -> str:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action in {"service.restart", "runtime.restart_api"}:
|
||||
return "restart"
|
||||
if normalized_action in {"service.start", "runtime.start_worker", "runtime.start_sync_agent"}:
|
||||
return "start"
|
||||
if normalized_action in {"service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"}:
|
||||
return "stop"
|
||||
return ""
|
||||
|
||||
|
||||
def execute_structured_action(
|
||||
action: str,
|
||||
payload: dict | None,
|
||||
*,
|
||||
service_names: dict[str, str],
|
||||
runner: CommandRunner,
|
||||
host_context: dict | None = None,
|
||||
) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
normalized_payload = dict(payload or {})
|
||||
normalized_service_names = {
|
||||
str(key or "").strip(): str(value or "").strip()
|
||||
for key, value in dict(service_names or {}).items()
|
||||
if str(key or "").strip() and str(value or "").strip()
|
||||
}
|
||||
host_details = {
|
||||
key: str((host_context or {}).get(key) or "").strip()
|
||||
for key in ("hostname", "ip")
|
||||
if str((host_context or {}).get(key) or "").strip()
|
||||
}
|
||||
|
||||
if normalized_action == "health.snapshot":
|
||||
checks: dict[str, dict] = {}
|
||||
for key, service_name in normalized_service_names.items():
|
||||
code, stdout, stderr = runner(["systemctl", "is-active", service_name], timeout=15)
|
||||
checks[key] = {
|
||||
"service_name": service_name,
|
||||
"returncode": int(code or 0),
|
||||
"state": stdout or stderr,
|
||||
"ok": int(code or 0) == 0 and (stdout or stderr) == "active",
|
||||
}
|
||||
result = {"checks": checks}
|
||||
result.update(host_details)
|
||||
return True, "health snapshot collected", result
|
||||
|
||||
if normalized_action == "service.status":
|
||||
service_name = service_name_from_payload(
|
||||
normalized_payload,
|
||||
default_worker_service_name=normalized_service_names.get("worker") or "",
|
||||
)
|
||||
code, stdout, stderr = runner(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
|
||||
result = {
|
||||
"service_name": service_name,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}
|
||||
result.update(host_details)
|
||||
return int(code or 0) == 0, stdout or stderr or f"{service_name} status collected", result
|
||||
|
||||
systemctl_action = systemctl_action_name(normalized_action)
|
||||
if systemctl_action:
|
||||
service_name = service_name_for_action(normalized_action, normalized_payload, normalized_service_names)
|
||||
if not service_name:
|
||||
return False, f"当前动作缺少 service_name: {normalized_action}", {"action": normalized_action}
|
||||
code, stdout, stderr = runner(["systemctl", systemctl_action, service_name], timeout=45)
|
||||
result = {
|
||||
"service_name": service_name,
|
||||
"systemctl_action": systemctl_action,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}
|
||||
result.update(host_details)
|
||||
return int(code or 0) == 0, stdout or stderr or f"{service_name} {systemctl_action} completed", result
|
||||
|
||||
if normalized_action == "logs.collect":
|
||||
service_name = service_name_from_payload(
|
||||
normalized_payload,
|
||||
default_worker_service_name=normalized_service_names.get("worker") or "",
|
||||
)
|
||||
lines = max(20, min(int(normalized_payload.get("lines") or 120), 500))
|
||||
code, stdout, stderr = runner(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
|
||||
journal_output = stdout or stderr
|
||||
fallback_output, fallback_paths = _collect_log_file_fallback(
|
||||
service_name,
|
||||
lines=lines,
|
||||
service_names=normalized_service_names,
|
||||
)
|
||||
collection_source = "journal"
|
||||
effective_output = journal_output
|
||||
ok = int(code or 0) == 0
|
||||
if (not ok or not stdout.strip()) and fallback_output:
|
||||
collection_source = "file_fallback"
|
||||
effective_output = fallback_output
|
||||
ok = True
|
||||
result = {
|
||||
"service_name": service_name,
|
||||
"lines": lines,
|
||||
"collection_source": collection_source,
|
||||
"fallback_used": bool(collection_source == "file_fallback"),
|
||||
"fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
|
||||
"log_paths": fallback_paths,
|
||||
"journal_returncode": int(code or 0),
|
||||
"stdout": trim_output(effective_output, 20000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}
|
||||
result.update(host_details)
|
||||
return ok, effective_output or f"{service_name} logs collected", result
|
||||
|
||||
if normalized_action == "diagnostics.collect":
|
||||
lines = max(20, min(int(normalized_payload.get("lines") or 200), 800))
|
||||
diagnostics: dict[str, object] = {
|
||||
"services": {},
|
||||
"lines": lines,
|
||||
}
|
||||
diagnostics.update(host_details)
|
||||
for key, service_name in normalized_service_names.items():
|
||||
active_code, active_stdout, active_stderr = runner(["systemctl", "is-active", service_name], timeout=15)
|
||||
status_code, status_stdout, status_stderr = runner(
|
||||
["systemctl", "status", service_name, "--no-pager", "-l"],
|
||||
timeout=45,
|
||||
)
|
||||
log_code, log_stdout, log_stderr = runner(
|
||||
["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"],
|
||||
timeout=45,
|
||||
)
|
||||
log_output = log_stdout or log_stderr
|
||||
fallback_output, fallback_paths = _collect_log_file_fallback(
|
||||
service_name,
|
||||
lines=lines,
|
||||
service_names=normalized_service_names,
|
||||
)
|
||||
log_source = "journal"
|
||||
if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
|
||||
log_output = fallback_output
|
||||
log_source = "file_fallback"
|
||||
diagnostics["services"][key] = {
|
||||
"service_name": service_name,
|
||||
"active_returncode": int(active_code or 0),
|
||||
"active_state": active_stdout or active_stderr,
|
||||
"status_returncode": int(status_code or 0),
|
||||
"status_output": trim_output(status_stdout or status_stderr, 12000),
|
||||
"log_returncode": int(log_code or 0),
|
||||
"log_source": log_source,
|
||||
"log_paths": fallback_paths,
|
||||
"log_output": trim_output(log_output, 20000),
|
||||
}
|
||||
return True, "diagnostics collected", diagnostics
|
||||
|
||||
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}
|
||||
2852
domain-api/app/services/ops_agent_service.py
Normal file
2852
domain-api/app/services/ops_agent_service.py
Normal file
File diff suppressed because it is too large
Load Diff
17
domain-api/app/services/ops_command_service.py
Normal file
17
domain-api/app/services/ops_command_service.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
OPS_MULTI_REGION_DIR = "domain-api/deploy/multi-region"
|
||||
|
||||
|
||||
def ops_script_path(script_name: str) -> str:
|
||||
normalized_script_name = str(script_name or "").strip().lstrip("/")
|
||||
if not normalized_script_name:
|
||||
return OPS_MULTI_REGION_DIR
|
||||
return f"{OPS_MULTI_REGION_DIR}/{normalized_script_name}"
|
||||
|
||||
|
||||
def build_bash_command(script_name: str, *args: object) -> str:
|
||||
command_parts = ["bash", ops_script_path(script_name)]
|
||||
command_parts.extend(str(arg or "").strip() for arg in args if str(arg or "").strip())
|
||||
return " ".join(command_parts)
|
||||
79
domain-api/app/services/ops_execution_capability_service.py
Normal file
79
domain-api/app/services/ops_execution_capability_service.py
Normal file
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.services.ops_action_executor_core import STRUCTURED_ACTIONS
|
||||
|
||||
|
||||
_LOCAL_RUNTIME_ONLY_ACTIONS = {
|
||||
"runtime.push_sync",
|
||||
"runtime.pull_tasks",
|
||||
}
|
||||
|
||||
_REMOTE_AGENT_ONLY_ACTIONS = {
|
||||
"delivery.queue.flush",
|
||||
"delivery.queue.replay",
|
||||
"delivery.queue.discard",
|
||||
}
|
||||
|
||||
_LOCAL_RUNTIME_ACTIONS = set(STRUCTURED_ACTIONS) | _LOCAL_RUNTIME_ONLY_ACTIONS | {"deploy.release"}
|
||||
_SSH_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
|
||||
_REMOTE_AGENT_ACTIONS = set(STRUCTURED_ACTIONS) | _REMOTE_AGENT_ONLY_ACTIONS | {"deploy.release"}
|
||||
_CONTROL_PLANE_ACTIONS = {
|
||||
"node.bootstrap",
|
||||
}
|
||||
|
||||
|
||||
def supports_local_runtime_action(action: str) -> bool:
|
||||
return str(action or "").strip() in _LOCAL_RUNTIME_ACTIONS
|
||||
|
||||
|
||||
def supports_ssh_action(action: str) -> bool:
|
||||
return str(action or "").strip() in _SSH_ACTIONS
|
||||
|
||||
|
||||
def supports_remote_agent_action(action: str) -> bool:
|
||||
return str(action or "").strip() in _REMOTE_AGENT_ACTIONS
|
||||
|
||||
|
||||
def supports_control_plane_action(action: str) -> bool:
|
||||
return str(action or "").strip() in _CONTROL_PLANE_ACTIONS
|
||||
|
||||
|
||||
def validate_action_execution_mode(
|
||||
action: str,
|
||||
execution_mode: str,
|
||||
*,
|
||||
target_node_code: str = "",
|
||||
current_node_code: str = "",
|
||||
) -> tuple[bool, str]:
|
||||
normalized_action = str(action or "").strip()
|
||||
normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
|
||||
normalized_target_node_code = str(target_node_code or "").strip()
|
||||
normalized_current_node_code = str(current_node_code or "").strip()
|
||||
|
||||
if normalized_mode == "local-runtime":
|
||||
if not supports_local_runtime_action(normalized_action):
|
||||
return False, f"{normalized_action} 当前不支持 local-runtime 执行方式。"
|
||||
if (
|
||||
normalized_target_node_code
|
||||
and normalized_current_node_code
|
||||
and normalized_target_node_code != normalized_current_node_code
|
||||
):
|
||||
return False, "local-runtime 仅支持当前控制面本机节点。"
|
||||
return True, ""
|
||||
|
||||
if normalized_mode == "control-plane":
|
||||
if not supports_control_plane_action(normalized_action):
|
||||
return False, f"{normalized_action} 当前不支持 control-plane 执行方式。"
|
||||
return True, ""
|
||||
|
||||
if normalized_mode == "ssh":
|
||||
if not supports_ssh_action(normalized_action):
|
||||
return False, f"{normalized_action} 当前不支持 ssh 执行方式。"
|
||||
return True, ""
|
||||
|
||||
if normalized_mode == "remote-agent":
|
||||
if not supports_remote_agent_action(normalized_action):
|
||||
return False, f"{normalized_action} 当前不支持 remote-agent 执行方式。"
|
||||
return True, ""
|
||||
|
||||
return False, f"未知执行方式: {normalized_mode}"
|
||||
60
domain-api/app/services/ops_execution_mode_service.py
Normal file
60
domain-api/app/services/ops_execution_mode_service.py
Normal file
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def normalize_execution_modes(raw_modes: object, fallback_mode: str = "remote-agent") -> list[str]:
|
||||
normalized_modes: list[str] = []
|
||||
seen: set[str] = set()
|
||||
fallback = str(fallback_mode or "remote-agent").strip() or "remote-agent"
|
||||
for item in list(raw_modes or []):
|
||||
mode = str(item or "").strip()
|
||||
if not mode or mode in seen:
|
||||
continue
|
||||
seen.add(mode)
|
||||
normalized_modes.append(mode)
|
||||
if fallback and fallback not in seen:
|
||||
normalized_modes.insert(0, fallback)
|
||||
return normalized_modes or [fallback]
|
||||
|
||||
|
||||
def execution_mode_label(execution_mode: str) -> str:
|
||||
normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
|
||||
if normalized_mode == "ssh":
|
||||
return "SSH"
|
||||
if normalized_mode == "control-plane":
|
||||
return "控制面"
|
||||
if normalized_mode == "local-runtime":
|
||||
return "本机运行时"
|
||||
if normalized_mode == "remote-agent":
|
||||
return "远端 Agent"
|
||||
return normalized_mode
|
||||
|
||||
|
||||
def build_execution_mode_options(raw_modes: object, fallback_mode: str = "remote-agent") -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"value": mode,
|
||||
"label": execution_mode_label(mode),
|
||||
}
|
||||
for mode in normalize_execution_modes(raw_modes, fallback_mode)
|
||||
]
|
||||
|
||||
|
||||
def decorate_execution_mode_fields(
|
||||
payload: dict | None = None,
|
||||
*,
|
||||
default_key: str = "default_execution_mode",
|
||||
modes_key: str = "execution_modes",
|
||||
) -> dict:
|
||||
normalized_payload = dict(payload or {})
|
||||
default_mode = str(normalized_payload.get(default_key) or "remote-agent").strip() or "remote-agent"
|
||||
execution_mode_options = build_execution_mode_options(normalized_payload.get(modes_key) or [], default_mode)
|
||||
normalized_payload[default_key] = default_mode
|
||||
normalized_payload[modes_key] = [str(item.get("value") or "").strip() for item in execution_mode_options if str(item.get("value") or "").strip()]
|
||||
normalized_payload["default_execution_mode_label"] = execution_mode_label(default_mode)
|
||||
normalized_payload["execution_mode_options"] = execution_mode_options
|
||||
normalized_payload["execution_mode_labels"] = [
|
||||
str(item.get("label") or "").strip()
|
||||
for item in execution_mode_options
|
||||
if str(item.get("label") or "").strip()
|
||||
]
|
||||
return normalized_payload
|
||||
1682
domain-api/app/services/ops_job_service.py
Normal file
1682
domain-api/app/services/ops_job_service.py
Normal file
File diff suppressed because it is too large
Load Diff
1263
domain-api/app/services/ops_playbook_service.py
Normal file
1263
domain-api/app/services/ops_playbook_service.py
Normal file
File diff suppressed because it is too large
Load Diff
494
domain-api/app/services/ops_policy_service.py
Normal file
494
domain-api/app/services/ops_policy_service.py
Normal file
@@ -0,0 +1,494 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.cluster_runtime_service import get_cluster_snapshot
|
||||
from app.services.ops_execution_capability_service import validate_action_execution_mode
|
||||
from app.services.ops_release_service import (
|
||||
_calculate_total_batches,
|
||||
_normalize_rollout_policy,
|
||||
_resolve_rollout_targets,
|
||||
build_release_rollout_gate,
|
||||
get_release,
|
||||
)
|
||||
|
||||
|
||||
_LOW_RISK_ACTIONS = {
|
||||
"health.snapshot",
|
||||
"runtime.push_sync",
|
||||
"runtime.pull_tasks",
|
||||
"logs.collect",
|
||||
"diagnostics.collect",
|
||||
"delivery.queue.flush",
|
||||
}
|
||||
|
||||
_MEDIUM_RISK_ACTIONS = {
|
||||
"runtime.start_worker",
|
||||
"runtime.start_sync_agent",
|
||||
"service.status",
|
||||
"health.check",
|
||||
"delivery.queue.replay",
|
||||
}
|
||||
|
||||
_HIGH_RISK_ACTIONS = {
|
||||
"runtime.stop_worker",
|
||||
"runtime.stop_sync_agent",
|
||||
"runtime.restart_api",
|
||||
"service.restart",
|
||||
"deploy.release",
|
||||
"delivery.queue.discard",
|
||||
}
|
||||
|
||||
_CRITICAL_RISK_ACTIONS = {
|
||||
"deploy.rollback",
|
||||
"node.bootstrap",
|
||||
"cluster.reconfigure",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_node_code_list(raw_value: object) -> list[str]:
|
||||
if isinstance(raw_value, list):
|
||||
return [str(item).strip() for item in raw_value if str(item).strip()]
|
||||
text = str(raw_value or "").replace("\r", "\n").strip()
|
||||
if not text:
|
||||
return []
|
||||
normalized = text.replace(",", "\n")
|
||||
return [item.strip() for item in normalized.split("\n") if item.strip()]
|
||||
|
||||
|
||||
def _dedupe_text_items(items: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip()))
|
||||
|
||||
|
||||
def _risk_level_for_action(action: str) -> str:
|
||||
if action in _LOW_RISK_ACTIONS:
|
||||
return "low"
|
||||
if action in _HIGH_RISK_ACTIONS:
|
||||
return "high"
|
||||
if action in _CRITICAL_RISK_ACTIONS:
|
||||
return "critical"
|
||||
return "medium"
|
||||
|
||||
|
||||
def _compact_target_node(item: dict) -> dict:
|
||||
return {
|
||||
"node_code": str(item.get("node_code") or ""),
|
||||
"region": str(item.get("region") or ""),
|
||||
"role": str(item.get("role") or ""),
|
||||
"status": str(item.get("status") or ""),
|
||||
"current_load": int(item.get("current_load", 0) or 0),
|
||||
"is_effective_worker": bool(item.get("is_effective_worker", False)),
|
||||
"detect_participating": bool(item.get("detect_participating", False)),
|
||||
"is_enabled": bool(item.get("is_enabled", True)),
|
||||
"deploy_channel": str(item.get("deploy_channel") or ""),
|
||||
}
|
||||
|
||||
|
||||
def _preview_action_payload_guardrails(
|
||||
action: str,
|
||||
payload: dict,
|
||||
*,
|
||||
execution_mode: str,
|
||||
target_nodes_total: int,
|
||||
) -> dict:
|
||||
warnings: list[str] = []
|
||||
blocking_reasons: list[str] = []
|
||||
approval_reasons: list[str] = []
|
||||
recommendations: list[str] = []
|
||||
|
||||
if action == "deploy.release":
|
||||
release_version = str(payload.get("release_version") or "").strip()
|
||||
artifact_url = str(payload.get("artifact_url") or "").strip()
|
||||
if not release_version:
|
||||
blocking_reasons.append("deploy.release 缺少 release_version,当前不能创建正式发布任务。")
|
||||
if not artifact_url:
|
||||
blocking_reasons.append("deploy.release 缺少 artifact_url,当前不能创建正式发布任务。")
|
||||
if execution_mode not in {"remote-agent", "ssh"}:
|
||||
blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。")
|
||||
if target_nodes_total > 1:
|
||||
recommendations.append("建议先单节点 canary 验证发布任务,再逐步扩到更多节点。")
|
||||
|
||||
if action == "node.bootstrap":
|
||||
if execution_mode != "control-plane":
|
||||
blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。")
|
||||
if target_nodes_total > 1:
|
||||
recommendations.append("接管动作建议按单节点节奏推进,先确认首台节点接入成功后再继续放量。")
|
||||
|
||||
if action in _CRITICAL_RISK_ACTIONS:
|
||||
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
|
||||
elif action in _HIGH_RISK_ACTIONS:
|
||||
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
|
||||
|
||||
return {
|
||||
"warnings": _dedupe_text_items(warnings),
|
||||
"blocking_reasons": _dedupe_text_items(blocking_reasons),
|
||||
"approval_reasons": _dedupe_text_items(approval_reasons),
|
||||
"recommendations": _dedupe_text_items(recommendations),
|
||||
}
|
||||
|
||||
|
||||
def _preview_single_node_policy(
|
||||
action: str,
|
||||
*,
|
||||
target_node_code: str,
|
||||
cluster: dict,
|
||||
payload: dict | None = None,
|
||||
execution_mode: str = "remote-agent",
|
||||
target_type: str = "node",
|
||||
include_payload_guardrails: bool = True,
|
||||
) -> dict:
|
||||
payload = dict(payload or {})
|
||||
nodes = list(cluster.get("nodes") or [])
|
||||
summary = cluster.get("summary") or {}
|
||||
target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {})
|
||||
|
||||
risk_level = _risk_level_for_action(action)
|
||||
warnings: list[str] = []
|
||||
blocking_reasons: list[str] = []
|
||||
approval_reasons: list[str] = []
|
||||
recommendations: list[str] = []
|
||||
|
||||
execution_supported, execution_reason = validate_action_execution_mode(
|
||||
action,
|
||||
execution_mode,
|
||||
target_node_code=target_node_code,
|
||||
current_node_code=settings.node_code,
|
||||
)
|
||||
if not execution_supported and execution_reason:
|
||||
blocking_reasons.append(execution_reason)
|
||||
|
||||
if target_type == "node" and target_node_code and not target_node:
|
||||
warnings.append("目标节点当前未出现在集群快照中,可能尚未注册或已离线。")
|
||||
|
||||
node_status = str(target_node.get("status") or "")
|
||||
node_role = str(target_node.get("role") or "")
|
||||
node_current_load = int(target_node.get("current_load", 0) or 0)
|
||||
node_effective_worker = bool(target_node.get("is_effective_worker", False))
|
||||
node_detect_participating = bool(target_node.get("detect_participating", False))
|
||||
|
||||
if target_node:
|
||||
if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
|
||||
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
|
||||
|
||||
if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
|
||||
blocking_reasons.append("目标节点正在参与检测,需先迁移负载或人工确认后再执行。")
|
||||
|
||||
if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
|
||||
approval_reasons.append("目标节点是 control 节点,建议强制走审批或维护窗口。")
|
||||
|
||||
if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}:
|
||||
blocking_reasons.append("当前有效执行节点数不足 2,执行该动作可能导致检测能力中断。")
|
||||
|
||||
if node_current_load > 0:
|
||||
warnings.append(f"目标节点当前负载为 {node_current_load},建议优先观察或转移任务。")
|
||||
|
||||
if include_payload_guardrails:
|
||||
payload_guardrails = _preview_action_payload_guardrails(
|
||||
action,
|
||||
payload,
|
||||
execution_mode=execution_mode,
|
||||
target_nodes_total=1,
|
||||
)
|
||||
warnings.extend(payload_guardrails.get("warnings") or [])
|
||||
blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or [])
|
||||
approval_reasons.extend(payload_guardrails.get("approval_reasons") or [])
|
||||
recommendations.extend(payload_guardrails.get("recommendations") or [])
|
||||
|
||||
warnings = _dedupe_text_items(warnings)
|
||||
blocking_reasons = _dedupe_text_items(blocking_reasons)
|
||||
approval_reasons = _dedupe_text_items(approval_reasons)
|
||||
recommendations = _dedupe_text_items(recommendations)
|
||||
|
||||
approval_required = bool(approval_reasons)
|
||||
blocked = bool(blocking_reasons)
|
||||
|
||||
return {
|
||||
"action": action,
|
||||
"target_type": target_type,
|
||||
"target_node_code": target_node_code,
|
||||
"execution_mode": execution_mode,
|
||||
"risk_level": risk_level,
|
||||
"approval_required": approval_required,
|
||||
"blocked": blocked,
|
||||
"blocking_reasons": blocking_reasons,
|
||||
"approval_reasons": approval_reasons,
|
||||
"warnings": warnings,
|
||||
"recommendations": recommendations,
|
||||
"target_node": {
|
||||
"node_code": str(target_node.get("node_code") or ""),
|
||||
"region": str(target_node.get("region") or ""),
|
||||
"role": node_role,
|
||||
"status": node_status,
|
||||
"current_load": node_current_load,
|
||||
"is_effective_worker": node_effective_worker,
|
||||
"detect_participating": node_detect_participating,
|
||||
} if target_node else {},
|
||||
"cluster_guardrails": {
|
||||
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
|
||||
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
|
||||
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
|
||||
"busy_nodes": list(summary.get("busy_nodes") or []),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict:
|
||||
target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes"))
|
||||
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||||
action_payload = dict(payload.get("payload") or {})
|
||||
summary = cluster.get("summary") or {}
|
||||
|
||||
warnings: list[str] = []
|
||||
blocking_reasons: list[str] = []
|
||||
approval_reasons: list[str] = []
|
||||
recommendations: list[str] = []
|
||||
|
||||
if not target_node_codes:
|
||||
blocking_reasons.append("当前没有可预检的目标节点。")
|
||||
|
||||
shared_guardrails = _preview_action_payload_guardrails(
|
||||
action,
|
||||
action_payload,
|
||||
execution_mode=execution_mode,
|
||||
target_nodes_total=len(target_node_codes),
|
||||
)
|
||||
warnings.extend(shared_guardrails.get("warnings") or [])
|
||||
blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or [])
|
||||
approval_reasons.extend(shared_guardrails.get("approval_reasons") or [])
|
||||
recommendations.extend(shared_guardrails.get("recommendations") or [])
|
||||
|
||||
node_previews = [
|
||||
_preview_single_node_policy(
|
||||
action,
|
||||
target_node_code=node_code,
|
||||
cluster=cluster,
|
||||
payload=action_payload,
|
||||
execution_mode=execution_mode,
|
||||
target_type="batch",
|
||||
include_payload_guardrails=False,
|
||||
)
|
||||
for node_code in target_node_codes
|
||||
]
|
||||
|
||||
blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))]
|
||||
approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))]
|
||||
warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])]
|
||||
missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")]
|
||||
target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"]
|
||||
|
||||
if blocked_nodes:
|
||||
blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断,需先处理后再批量执行。")
|
||||
if approval_nodes:
|
||||
approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。")
|
||||
if warning_nodes:
|
||||
warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。")
|
||||
if missing_nodes:
|
||||
warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。")
|
||||
if target_control_nodes and len(target_node_codes) > 1:
|
||||
recommendations.append("建议把 control 节点放到批次尾部,优先在 worker 或空闲节点验证。")
|
||||
|
||||
warnings = _dedupe_text_items(warnings)
|
||||
blocking_reasons = _dedupe_text_items(blocking_reasons)
|
||||
approval_reasons = _dedupe_text_items(approval_reasons)
|
||||
recommendations = _dedupe_text_items(recommendations)
|
||||
|
||||
return {
|
||||
"action": action,
|
||||
"target_type": "batch",
|
||||
"execution_mode": execution_mode,
|
||||
"risk_level": _risk_level_for_action(action),
|
||||
"approval_required": bool(approval_reasons),
|
||||
"blocked": bool(blocking_reasons),
|
||||
"blocking_reasons": blocking_reasons,
|
||||
"approval_reasons": approval_reasons,
|
||||
"warnings": warnings,
|
||||
"recommendations": recommendations,
|
||||
"target_summary": {
|
||||
"nodes_total": len(target_node_codes),
|
||||
"blocked_nodes": len(blocked_nodes),
|
||||
"approval_nodes": len(approval_nodes),
|
||||
"warning_nodes": len(warning_nodes),
|
||||
"missing_nodes": len(missing_nodes),
|
||||
},
|
||||
"target_nodes": [
|
||||
{
|
||||
**(item.get("target_node") or {}),
|
||||
"node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""),
|
||||
"risk_level": str(item.get("risk_level") or ""),
|
||||
"approval_required": bool(item.get("approval_required", False)),
|
||||
"blocked": bool(item.get("blocked", False)),
|
||||
"blocking_reasons": list(item.get("blocking_reasons") or []),
|
||||
"approval_reasons": list(item.get("approval_reasons") or []),
|
||||
"warnings": list(item.get("warnings") or []),
|
||||
}
|
||||
for item in node_previews
|
||||
],
|
||||
"cluster_guardrails": {
|
||||
"online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
|
||||
"dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
|
||||
"online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
|
||||
"busy_nodes": list(summary.get("busy_nodes") or []),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict:
|
||||
selector = payload.get("target_selector") or {}
|
||||
input_policy = payload.get("policy") or {}
|
||||
release_id = int(payload.get("release_id") or 0)
|
||||
input_release = dict(payload.get("release") or {})
|
||||
release = input_release or (get_release(release_id) if release_id > 0 else {})
|
||||
target_nodes = _resolve_rollout_targets(selector)
|
||||
normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes))
|
||||
execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||||
summary = cluster.get("summary") or {}
|
||||
release_gate = build_release_rollout_gate(
|
||||
release,
|
||||
target_nodes,
|
||||
execution_mode=execution_mode,
|
||||
)
|
||||
operational_readiness = dict(release_gate.get("operational_readiness") or {})
|
||||
|
||||
risk_level = _risk_level_for_action(action)
|
||||
warnings: list[str] = []
|
||||
blocking_reasons: list[str] = []
|
||||
approval_reasons: list[str] = []
|
||||
recommendations: list[str] = []
|
||||
|
||||
target_count = len(target_nodes)
|
||||
first_batch_size = int(normalized_policy.get("first_batch_size") or 0)
|
||||
batch_size = int(normalized_policy.get("batch_size") or 0)
|
||||
batches_total = _calculate_total_batches(target_count, normalized_policy)
|
||||
first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else []
|
||||
|
||||
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
|
||||
dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0)
|
||||
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
|
||||
|
||||
target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"]
|
||||
target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"]
|
||||
target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))]
|
||||
first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))]
|
||||
first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"]
|
||||
|
||||
busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0]
|
||||
participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))]
|
||||
offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}]
|
||||
|
||||
if target_count <= 0:
|
||||
blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。")
|
||||
if release_id > 0 and not input_release and not release:
|
||||
blocking_reasons.append("指定的 Release 不存在,当前无法预检该 rollout。")
|
||||
gate_status = str(release_gate.get("status") or "").strip()
|
||||
if gate_status in {"release_not_ready", "artifact_missing"}:
|
||||
warnings.append(str(release_gate.get("summary") or "").strip())
|
||||
|
||||
if offline_targets:
|
||||
warnings.append(f"命中了 {len(offline_targets)} 个非在线节点,若保留 only_online=false,发布时可能出现排队或失败。")
|
||||
|
||||
if busy_targets:
|
||||
warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点,建议优先确认任务是否可中断。")
|
||||
|
||||
if participating_targets:
|
||||
warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点,建议优先滚动到空闲节点。")
|
||||
|
||||
if target_control_nodes:
|
||||
approval_reasons.append("本次 rollout 覆盖 control 节点,建议强制审批,并把 control 放在最后一批。")
|
||||
|
||||
if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes:
|
||||
blocking_reasons.append("第一批会覆盖全部有效执行节点,检测能力可能瞬时归零。请缩小 first_batch_size。")
|
||||
|
||||
if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes:
|
||||
blocking_reasons.append("第一批会覆盖全部在线控制面,运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。")
|
||||
|
||||
if first_batch_size > 1 and target_control_nodes:
|
||||
warnings.append("当前第一批大于 1,且目标中包含 control 节点;默认建议先 worker、后 control。")
|
||||
|
||||
if action in _CRITICAL_RISK_ACTIONS:
|
||||
approval_reasons.append("该动作属于 critical 风险动作,正式环境必须审批。")
|
||||
elif action in _HIGH_RISK_ACTIONS:
|
||||
approval_reasons.append("该动作属于 high 风险动作,建议审批后执行。")
|
||||
|
||||
blocking_reasons.extend(list(release_gate.get("blocking_reasons") or []))
|
||||
warnings.extend(list(release_gate.get("warning_reasons") or []))
|
||||
recommendations.extend(list(release_gate.get("recommendations") or []))
|
||||
|
||||
readiness_summary = dict(operational_readiness.get("summary") or {})
|
||||
if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count:
|
||||
approval_reasons.append("本次目标中存在巡检未完全通过的节点,建议先完成标准巡检并经人工确认后再发布。")
|
||||
|
||||
if target_count > 1 and first_batch_size > 1:
|
||||
recommendations.append("正式环境建议 first_batch_size=1,从单节点 canary 开始。")
|
||||
if target_control_nodes:
|
||||
recommendations.append("建议把 control 节点放在 rollout 末尾,并确保 worker 已先通过健康检查。")
|
||||
if busy_targets or participating_targets:
|
||||
recommendations.append("建议先等参与检测的节点降为空闲,再推进该批次。")
|
||||
|
||||
warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip()))
|
||||
blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip()))
|
||||
approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip()))
|
||||
recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip()))
|
||||
|
||||
approval_required = bool(approval_reasons)
|
||||
blocked = bool(blocking_reasons)
|
||||
|
||||
return {
|
||||
"action": action,
|
||||
"target_type": "rollout",
|
||||
"risk_level": risk_level,
|
||||
"approval_required": approval_required,
|
||||
"blocked": blocked,
|
||||
"blocking_reasons": blocking_reasons,
|
||||
"approval_reasons": approval_reasons,
|
||||
"warnings": warnings,
|
||||
"recommendations": recommendations,
|
||||
"target_summary": {
|
||||
"nodes_total": target_count,
|
||||
"worker_nodes": len(target_worker_nodes),
|
||||
"control_nodes": len(target_control_nodes),
|
||||
"effective_worker_nodes": len(target_effective_workers),
|
||||
"busy_nodes": len(busy_targets),
|
||||
"participating_nodes": len(participating_targets),
|
||||
"offline_nodes": len(offline_targets),
|
||||
},
|
||||
"batch_plan": {
|
||||
"first_batch_size": first_batch_size,
|
||||
"batch_size": batch_size,
|
||||
"batches_total": batches_total,
|
||||
"first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets],
|
||||
"remaining_after_first_batch": max(target_count - len(first_batch_targets), 0),
|
||||
},
|
||||
"cluster_guardrails": {
|
||||
"online_worker_nodes": online_worker_nodes,
|
||||
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
|
||||
"online_control_nodes": online_control_nodes,
|
||||
"busy_nodes": list(summary.get("busy_nodes") or []),
|
||||
},
|
||||
"release": release,
|
||||
"release_gate": release_gate,
|
||||
"target_nodes": [_compact_target_node(item) for item in target_nodes],
|
||||
"operational_readiness": operational_readiness,
|
||||
}
|
||||
|
||||
|
||||
def preview_ops_job_policy(payload: dict) -> dict:
|
||||
action = str(payload.get("action") or "").strip()
|
||||
target_type = str(payload.get("target_type") or "node").strip() or "node"
|
||||
target_node_code = str(payload.get("target_node_code") or "").strip()
|
||||
execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||||
action_payload = dict(payload.get("payload") or {})
|
||||
|
||||
cluster = get_cluster_snapshot()
|
||||
if target_type == "rollout":
|
||||
return _preview_rollout_policy(action, payload, cluster)
|
||||
if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []):
|
||||
return _preview_node_batch_policy(action, payload, cluster)
|
||||
|
||||
return _preview_single_node_policy(
|
||||
action,
|
||||
target_node_code=target_node_code,
|
||||
cluster=cluster,
|
||||
payload=action_payload,
|
||||
execution_mode=execution_mode,
|
||||
target_type=target_type,
|
||||
include_payload_guardrails=True,
|
||||
)
|
||||
475
domain-api/app/services/ops_release_executor_core.py
Normal file
475
domain-api/app/services/ops_release_executor_core.py
Normal file
@@ -0,0 +1,475 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import shutil
|
||||
import tarfile
|
||||
import textwrap
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def normalize_text_list(raw_value: object) -> list[str]:
|
||||
if isinstance(raw_value, list):
|
||||
return [str(item).strip() for item in raw_value if str(item).strip()]
|
||||
text = str(raw_value or "").replace("\r", "\n").strip()
|
||||
if not text:
|
||||
return []
|
||||
normalized = text.replace(",", "\n")
|
||||
return [item.strip() for item in normalized.split("\n") if item.strip()]
|
||||
|
||||
|
||||
def coerce_bool(raw_value: object, default: bool = False) -> bool:
|
||||
if raw_value is None:
|
||||
return bool(default)
|
||||
if isinstance(raw_value, bool):
|
||||
return raw_value
|
||||
if isinstance(raw_value, (int, float)):
|
||||
return bool(raw_value)
|
||||
normalized = str(raw_value or "").strip().lower()
|
||||
if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
|
||||
return False
|
||||
return bool(default) if normalized == "" else bool(normalized)
|
||||
|
||||
|
||||
def normalize_release_health_check_services(
|
||||
payload: dict,
|
||||
restart_services: list[str],
|
||||
*,
|
||||
default_api_service_name: str,
|
||||
) -> list[str]:
|
||||
raw_health_check_services = payload.get("health_check_services")
|
||||
if raw_health_check_services is None:
|
||||
health_check_service_source = restart_services or [default_api_service_name]
|
||||
else:
|
||||
health_check_service_source = raw_health_check_services
|
||||
return normalize_text_list(health_check_service_source)
|
||||
|
||||
|
||||
def collect_service_state(run_command, service_name: str) -> dict:
|
||||
code, stdout, stderr = run_command(["systemctl", "is-active", service_name], timeout=15)
|
||||
state = stdout or stderr
|
||||
return {
|
||||
"service_name": service_name,
|
||||
"returncode": int(code or 0),
|
||||
"state": state,
|
||||
"ok": int(code or 0) == 0 and state == "active",
|
||||
}
|
||||
|
||||
|
||||
def check_health_url(url: str, timeout: int, *, user_agent: str, urlopen_func) -> dict:
|
||||
request = urllib.request.Request(
|
||||
url=str(url).strip(),
|
||||
headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urlopen_func(request, timeout=timeout) as response:
|
||||
body = response.read(4000).decode("utf-8", errors="ignore")
|
||||
status_code = int(getattr(response, "status", 0) or response.getcode() or 0)
|
||||
return {
|
||||
"url": str(url).strip(),
|
||||
"ok": 200 <= status_code < 400,
|
||||
"http_status": status_code,
|
||||
"body_preview": body[-1000:],
|
||||
}
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="ignore") if exc.fp else ""
|
||||
return {
|
||||
"url": str(url).strip(),
|
||||
"ok": False,
|
||||
"http_status": int(exc.code or 0),
|
||||
"body_preview": body[-1000:],
|
||||
"error": str(exc),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"url": str(url).strip(),
|
||||
"ok": False,
|
||||
"http_status": 0,
|
||||
"body_preview": "",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def run_release_health_checks(
|
||||
*,
|
||||
urls: list[str],
|
||||
services: list[str],
|
||||
timeout: int,
|
||||
retries: int,
|
||||
interval_seconds: int,
|
||||
run_command,
|
||||
user_agent: str,
|
||||
urlopen_func,
|
||||
) -> tuple[bool, dict]:
|
||||
normalized_urls = [str(item).strip() for item in urls if str(item).strip()]
|
||||
normalized_services = [str(item).strip() for item in services if str(item).strip()]
|
||||
attempts: list[dict] = []
|
||||
total_attempts = max(1, retries + 1)
|
||||
|
||||
for attempt_index in range(1, total_attempts + 1):
|
||||
service_results = [collect_service_state(run_command, service_name) for service_name in normalized_services]
|
||||
url_results = [
|
||||
check_health_url(url, timeout=timeout, user_agent=user_agent, urlopen_func=urlopen_func)
|
||||
for url in normalized_urls
|
||||
]
|
||||
services_ok = all(bool(item.get("ok", False)) for item in service_results) if service_results else True
|
||||
urls_ok = all(bool(item.get("ok", False)) for item in url_results) if url_results else True
|
||||
attempt_payload = {
|
||||
"attempt": attempt_index,
|
||||
"services": service_results,
|
||||
"urls": url_results,
|
||||
"ok": services_ok and urls_ok,
|
||||
}
|
||||
attempts.append(attempt_payload)
|
||||
if attempt_payload["ok"]:
|
||||
return True, {
|
||||
"ok": True,
|
||||
"attempts": attempts,
|
||||
"services_checked": normalized_services,
|
||||
"urls_checked": normalized_urls,
|
||||
}
|
||||
if attempt_index < total_attempts:
|
||||
time.sleep(max(0, interval_seconds))
|
||||
|
||||
return False, {
|
||||
"ok": False,
|
||||
"attempts": attempts,
|
||||
"services_checked": normalized_services,
|
||||
"urls_checked": normalized_urls,
|
||||
}
|
||||
|
||||
|
||||
def safe_extract_tar(archive: tarfile.TarFile, target_dir: Path) -> None:
|
||||
target_dir_resolved = target_dir.resolve()
|
||||
members = archive.getmembers()
|
||||
for member in members:
|
||||
member_path = (target_dir / member.name).resolve()
|
||||
if not str(member_path).startswith(str(target_dir_resolved)):
|
||||
raise RuntimeError(f"unsafe archive member: {member.name}")
|
||||
try:
|
||||
archive.extractall(target_dir, members=members, filter="data")
|
||||
except TypeError:
|
||||
archive.extractall(target_dir, members=members)
|
||||
|
||||
|
||||
def execute_release_action(
|
||||
payload: dict,
|
||||
*,
|
||||
run_command,
|
||||
default_api_service_name: str,
|
||||
event_callback=None,
|
||||
urlopen_func=None,
|
||||
user_agent: str = "domaincheck-ops/0.1",
|
||||
) -> tuple[bool, str, dict]:
|
||||
normalized_payload = dict(payload or {})
|
||||
event_callback = event_callback or (lambda event_type, message, level="info", payload=None: None)
|
||||
urlopen_func = urlopen_func or urllib.request.urlopen
|
||||
|
||||
release_version = str(normalized_payload.get("release_version") or "").strip()
|
||||
artifact_url = str(normalized_payload.get("artifact_url") or "").strip()
|
||||
checksum = str(normalized_payload.get("checksum") or "").strip().lower()
|
||||
install_root = Path(str(normalized_payload.get("install_root") or "/opt/domaincheck")).resolve()
|
||||
switch_current = coerce_bool(normalized_payload.get("switch_current", True), default=True)
|
||||
restart_services = normalize_text_list(normalized_payload.get("restart_services"))
|
||||
health_check_urls = normalize_text_list(normalized_payload.get("health_check_urls"))
|
||||
health_check_services = normalize_release_health_check_services(
|
||||
normalized_payload,
|
||||
restart_services,
|
||||
default_api_service_name=default_api_service_name,
|
||||
)
|
||||
health_check_timeout_seconds = max(2, int(normalized_payload.get("health_check_timeout_seconds") or 10))
|
||||
health_check_retries = max(0, int(normalized_payload.get("health_check_retries") or 2))
|
||||
health_check_interval_seconds = max(0, int(normalized_payload.get("health_check_interval_seconds") or 2))
|
||||
rollback_on_failure = coerce_bool(normalized_payload.get("rollback_on_failure", True), default=True)
|
||||
|
||||
if not release_version:
|
||||
return False, "release_version missing", {}
|
||||
if not artifact_url:
|
||||
return False, "artifact_url missing", {}
|
||||
|
||||
downloads_dir = install_root / "downloads"
|
||||
releases_dir = install_root / "releases"
|
||||
target_dir = releases_dir / release_version
|
||||
temp_dir = releases_dir / f"{release_version}.tmp"
|
||||
artifact_path = downloads_dir / f"{release_version}.tar.gz"
|
||||
current_link = install_root / "current"
|
||||
previous_current_target = ""
|
||||
downloads_dir.mkdir(parents=True, exist_ok=True)
|
||||
releases_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if current_link.exists():
|
||||
try:
|
||||
previous_current_target = str(current_link.resolve(strict=True))
|
||||
except Exception:
|
||||
try:
|
||||
previous_current_target = str(current_link.resolve())
|
||||
except Exception:
|
||||
previous_current_target = ""
|
||||
|
||||
event_callback(
|
||||
"deploy_download_started",
|
||||
f"开始下载发布包: {release_version}",
|
||||
payload={"artifact_url": artifact_url},
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
url=artifact_url,
|
||||
headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
|
||||
method="GET",
|
||||
)
|
||||
with urlopen_func(request, timeout=120) as response:
|
||||
artifact_bytes = response.read()
|
||||
artifact_path.write_bytes(artifact_bytes)
|
||||
event_callback(
|
||||
"deploy_download_completed",
|
||||
f"发布包下载完成: {release_version}",
|
||||
payload={"artifact_path": str(artifact_path), "size_bytes": len(artifact_bytes)},
|
||||
)
|
||||
|
||||
calculated_checksum = hashlib.sha256(artifact_bytes).hexdigest().lower()
|
||||
if checksum and checksum != calculated_checksum:
|
||||
return False, "release checksum mismatch", {
|
||||
"expected_checksum": checksum,
|
||||
"calculated_checksum": calculated_checksum,
|
||||
}
|
||||
event_callback(
|
||||
"deploy_checksum_verified",
|
||||
f"发布包校验通过: {release_version}",
|
||||
payload={"checksum": calculated_checksum},
|
||||
)
|
||||
|
||||
if temp_dir.exists():
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(artifact_path, "r:gz") as archive:
|
||||
safe_extract_tar(archive, temp_dir)
|
||||
|
||||
if target_dir.exists():
|
||||
extracted_target = target_dir
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
else:
|
||||
temp_dir.rename(target_dir)
|
||||
extracted_target = target_dir
|
||||
|
||||
meta_path = extracted_target / ".release-meta.json"
|
||||
meta_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"release_version": release_version,
|
||||
"artifact_url": artifact_url,
|
||||
"checksum": calculated_checksum,
|
||||
"deployed_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if switch_current:
|
||||
if current_link.is_symlink() or current_link.is_file():
|
||||
current_link.unlink(missing_ok=True)
|
||||
elif current_link.exists():
|
||||
raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
|
||||
current_link.symlink_to(extracted_target)
|
||||
event_callback(
|
||||
"deploy_current_switched",
|
||||
f"current 已切换到 {release_version}",
|
||||
payload={"current_link": str(current_link), "target": str(extracted_target)},
|
||||
)
|
||||
|
||||
restarted: list[dict] = []
|
||||
for service_name in restart_services:
|
||||
normalized_service_name = str(service_name or "").strip()
|
||||
if not normalized_service_name:
|
||||
continue
|
||||
event_callback(
|
||||
"deploy_service_restart",
|
||||
f"重启服务: {normalized_service_name}",
|
||||
payload={"service_name": normalized_service_name},
|
||||
)
|
||||
code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
|
||||
restarted.append(
|
||||
{
|
||||
"service_name": normalized_service_name,
|
||||
"returncode": int(code or 0),
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
}
|
||||
)
|
||||
if int(code or 0) != 0:
|
||||
return False, f"restart failed: {normalized_service_name}", {
|
||||
"release_version": release_version,
|
||||
"release_dir": str(extracted_target),
|
||||
"artifact_path": str(artifact_path),
|
||||
"checksum": calculated_checksum,
|
||||
"previous_current_target": previous_current_target,
|
||||
"restart_results": restarted,
|
||||
}
|
||||
|
||||
health_ok, health_result = run_release_health_checks(
|
||||
urls=health_check_urls,
|
||||
services=health_check_services,
|
||||
timeout=health_check_timeout_seconds,
|
||||
retries=health_check_retries,
|
||||
interval_seconds=health_check_interval_seconds,
|
||||
run_command=run_command,
|
||||
user_agent=user_agent,
|
||||
urlopen_func=urlopen_func,
|
||||
)
|
||||
if not health_ok:
|
||||
rollback_result = {
|
||||
"attempted": rollback_on_failure,
|
||||
"restored_to": previous_current_target,
|
||||
"restart_results": [],
|
||||
"health_result": health_result,
|
||||
}
|
||||
event_callback(
|
||||
"deploy_health_failed",
|
||||
f"发布健康检查失败: {release_version}",
|
||||
level="error",
|
||||
payload=health_result,
|
||||
)
|
||||
if rollback_on_failure and previous_current_target:
|
||||
if current_link.is_symlink() or current_link.is_file():
|
||||
current_link.unlink(missing_ok=True)
|
||||
elif current_link.exists():
|
||||
raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
|
||||
current_link.symlink_to(Path(previous_current_target))
|
||||
for service_name in restart_services:
|
||||
normalized_service_name = str(service_name or "").strip()
|
||||
if not normalized_service_name:
|
||||
continue
|
||||
code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
|
||||
rollback_result["restart_results"].append(
|
||||
{
|
||||
"service_name": normalized_service_name,
|
||||
"returncode": int(code or 0),
|
||||
"stdout": stdout,
|
||||
"stderr": stderr,
|
||||
}
|
||||
)
|
||||
rollback_result["post_rollback_health"] = run_release_health_checks(
|
||||
urls=health_check_urls,
|
||||
services=health_check_services,
|
||||
timeout=health_check_timeout_seconds,
|
||||
retries=0,
|
||||
interval_seconds=0,
|
||||
run_command=run_command,
|
||||
user_agent=user_agent,
|
||||
urlopen_func=urlopen_func,
|
||||
)[1]
|
||||
event_callback(
|
||||
"deploy_rollback_completed",
|
||||
f"已回滚到旧版本: {previous_current_target}",
|
||||
level="warning",
|
||||
payload=rollback_result,
|
||||
)
|
||||
return False, "release health check failed", {
|
||||
"release_version": release_version,
|
||||
"release_dir": str(extracted_target),
|
||||
"artifact_path": str(artifact_path),
|
||||
"checksum": calculated_checksum,
|
||||
"current_link": str(current_link),
|
||||
"previous_current_target": previous_current_target,
|
||||
"restart_results": restarted,
|
||||
"health_check": health_result,
|
||||
"rollback": rollback_result,
|
||||
}
|
||||
|
||||
event_callback(
|
||||
"deploy_health_passed",
|
||||
f"发布健康检查通过: {release_version}",
|
||||
payload=health_result,
|
||||
)
|
||||
return True, "release deployed", {
|
||||
"release_version": release_version,
|
||||
"release_dir": str(extracted_target),
|
||||
"artifact_path": str(artifact_path),
|
||||
"checksum": calculated_checksum,
|
||||
"current_link": str(current_link),
|
||||
"previous_current_target": previous_current_target,
|
||||
"restart_results": restarted,
|
||||
"health_check": health_result,
|
||||
}
|
||||
|
||||
|
||||
def build_remote_release_action_script(
|
||||
payload: dict,
|
||||
*,
|
||||
default_api_service_name: str = "domaincheck-api",
|
||||
user_agent: str = "domaincheck-ssh/0.1",
|
||||
) -> str:
|
||||
helper_functions = [
|
||||
normalize_text_list,
|
||||
coerce_bool,
|
||||
normalize_release_health_check_services,
|
||||
collect_service_state,
|
||||
check_health_url,
|
||||
run_release_health_checks,
|
||||
safe_extract_tar,
|
||||
execute_release_action,
|
||||
]
|
||||
helper_source = "\n\n".join(
|
||||
textwrap.dedent(inspect.getsource(func)).strip("\n")
|
||||
for func in helper_functions
|
||||
)
|
||||
return f"""from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import tarfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
{helper_source}
|
||||
|
||||
|
||||
PAYLOAD = {json.dumps(dict(payload or {{}}), ensure_ascii=False)}
|
||||
DEFAULT_API_SERVICE_NAME = {json.dumps(str(default_api_service_name or 'domaincheck-api'), ensure_ascii=False)}
|
||||
USER_AGENT = {json.dumps(str(user_agent or 'domaincheck-ssh/0.1'), ensure_ascii=False)}
|
||||
|
||||
|
||||
def _run(command, timeout=60):
|
||||
import subprocess
|
||||
|
||||
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
|
||||
|
||||
|
||||
def _event_callback(event_type, message, level="info", payload=None):
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
ok, message, result = execute_release_action(
|
||||
PAYLOAD,
|
||||
run_command=_run,
|
||||
default_api_service_name=DEFAULT_API_SERVICE_NAME,
|
||||
event_callback=_event_callback,
|
||||
urlopen_func=urllib.request.urlopen,
|
||||
user_agent=USER_AGENT,
|
||||
)
|
||||
except Exception as exc:
|
||||
ok, message, result = False, str(exc), {{"exception": str(exc), "action": "deploy.release"}}
|
||||
print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
"""
|
||||
3448
domain-api/app/services/ops_release_service.py
Normal file
3448
domain-api/app/services/ops_release_service.py
Normal file
File diff suppressed because it is too large
Load Diff
420
domain-api/app/services/ops_runtime_executor_service.py
Normal file
420
domain-api/app/services/ops_runtime_executor_service.py
Normal file
@@ -0,0 +1,420 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_action_executor_core import (
|
||||
STRUCTURED_ACTIONS,
|
||||
build_service_name_map,
|
||||
execute_structured_action,
|
||||
trim_output,
|
||||
)
|
||||
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
|
||||
|
||||
|
||||
_SSH_SUPPORTED_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
|
||||
_SSH_CONNECT_TIMEOUT_SECONDS = 12
|
||||
_SSH_REMOTE_TIMEOUT_SECONDS = {
|
||||
"health.snapshot": 30,
|
||||
"service.status": 45,
|
||||
"service.restart": 45,
|
||||
"service.start": 45,
|
||||
"service.stop": 45,
|
||||
"logs.collect": 45,
|
||||
"diagnostics.collect": 90,
|
||||
"runtime.start_worker": 45,
|
||||
"runtime.stop_worker": 45,
|
||||
"runtime.restart_api": 45,
|
||||
"runtime.start_sync_agent": 45,
|
||||
"runtime.stop_sync_agent": 45,
|
||||
"deploy.release": 180,
|
||||
}
|
||||
|
||||
|
||||
def supports_ssh_execution(action: str) -> bool:
|
||||
return str(action or "").strip() in _SSH_SUPPORTED_ACTIONS
|
||||
|
||||
|
||||
def execute_local_support_action(action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
|
||||
return False, f"当前未实现本机支持动作: {normalized_action}", {}
|
||||
if normalized_action == "deploy.release":
|
||||
return execute_release_action(
|
||||
dict(payload or {}),
|
||||
run_command=_run_local_command,
|
||||
default_api_service_name=settings.api_service_name,
|
||||
user_agent="domaincheck-local-runtime/0.1",
|
||||
)
|
||||
return execute_structured_action(
|
||||
normalized_action,
|
||||
dict(payload or {}),
|
||||
service_names=_service_name_map(),
|
||||
runner=_run_local_command,
|
||||
)
|
||||
|
||||
|
||||
def execute_ssh_action(node: dict, action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
|
||||
return False, f"当前 SSH 执行器暂不支持动作: {normalized_action}", {}
|
||||
|
||||
ssh_host = str(node.get("ssh_host") or "").strip()
|
||||
ssh_user = str(node.get("ssh_user") or "").strip()
|
||||
ssh_port = max(1, int(node.get("ssh_port") or 22))
|
||||
node_code = str(node.get("node_code") or "").strip()
|
||||
if not ssh_host or not ssh_user:
|
||||
return False, "目标节点缺少 SSH 主机或用户信息", {
|
||||
"node_code": node_code,
|
||||
"ssh_host": ssh_host,
|
||||
"ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port,
|
||||
}
|
||||
|
||||
normalized_payload = dict(payload or {})
|
||||
service_names = _service_name_map()
|
||||
if normalized_action == "deploy.release":
|
||||
remote_script = build_remote_release_action_script(
|
||||
normalized_payload,
|
||||
default_api_service_name=service_names.get("api") or "domaincheck-api",
|
||||
user_agent="domaincheck-ssh/0.1",
|
||||
)
|
||||
else:
|
||||
remote_script = _build_remote_python_script(
|
||||
action=normalized_action,
|
||||
payload=normalized_payload,
|
||||
service_names=service_names,
|
||||
)
|
||||
remote_command = "\n".join(
|
||||
[
|
||||
"set -euo pipefail",
|
||||
"if command -v python3 >/dev/null 2>&1; then",
|
||||
" PYTHON_BIN=python3",
|
||||
"elif command -v python >/dev/null 2>&1; then",
|
||||
" PYTHON_BIN=python",
|
||||
"else",
|
||||
' echo "{\\"ok\\": false, \\"message\\": \\"python interpreter missing on remote node\\", \\"result\\": {\\"executor\\": \\"ssh\\"}}"',
|
||||
" exit 127",
|
||||
"fi",
|
||||
'"${PYTHON_BIN}" - <<\'PY\'',
|
||||
remote_script.rstrip("\n"),
|
||||
"PY",
|
||||
]
|
||||
)
|
||||
ssh_command = [
|
||||
"ssh",
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"PreferredAuthentications=publickey",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=accept-new",
|
||||
"-o",
|
||||
f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
|
||||
"-p",
|
||||
str(ssh_port),
|
||||
f"{ssh_user}@{ssh_host}",
|
||||
remote_command,
|
||||
]
|
||||
timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS
|
||||
completed = subprocess.run(
|
||||
ssh_command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
transport = {
|
||||
"executor": "ssh",
|
||||
"node_code": node_code,
|
||||
"ssh_host": ssh_host,
|
||||
"ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port,
|
||||
"action": normalized_action,
|
||||
"returncode": int(completed.returncode or 0),
|
||||
}
|
||||
parsed_ok, parsed_payload = _extract_json_from_output(completed.stdout or "")
|
||||
if parsed_ok:
|
||||
ok = bool(parsed_payload.get("ok", False))
|
||||
message = str(parsed_payload.get("message") or "").strip() or ("SSH 动作执行成功" if ok else "SSH 动作执行失败")
|
||||
result = dict(parsed_payload.get("result") or {})
|
||||
result["transport"] = {
|
||||
**transport,
|
||||
"stdout_preview": trim_output(completed.stdout, 4000),
|
||||
"stderr_preview": trim_output(completed.stderr, 2000),
|
||||
}
|
||||
if completed.returncode != 0 and ok:
|
||||
ok = False
|
||||
message = f"SSH 命令返回码异常: {completed.returncode}"
|
||||
return ok, message, result
|
||||
|
||||
stderr_preview = trim_output(completed.stderr, 4000)
|
||||
stdout_preview = trim_output(completed.stdout, 4000)
|
||||
message = stderr_preview or stdout_preview or f"SSH 执行失败,返回码 {completed.returncode}"
|
||||
return False, message, {
|
||||
"executor": "ssh",
|
||||
"action": normalized_action,
|
||||
"transport": {
|
||||
**transport,
|
||||
"stdout_preview": stdout_preview,
|
||||
"stderr_preview": stderr_preview,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _service_name_map() -> dict[str, str]:
|
||||
return build_service_name_map(
|
||||
api_service_name=settings.api_service_name,
|
||||
worker_service_name=settings.worker_service_name,
|
||||
sync_agent_service_name=settings.sync_agent_service_name,
|
||||
)
|
||||
|
||||
|
||||
def _run_local_command(command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
|
||||
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
|
||||
|
||||
|
||||
def _build_remote_python_script(*, action: str, payload: dict, service_names: dict[str, str]) -> str:
|
||||
return f"""import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SERVICE_NAMES = {json.dumps(service_names, ensure_ascii=False)}
|
||||
ACTION = {json.dumps(str(action or '').strip(), ensure_ascii=False)}
|
||||
PAYLOAD = {json.dumps(dict(payload or {}), ensure_ascii=False)}
|
||||
|
||||
|
||||
def trim_output(text, limit):
|
||||
value = str(text or "").strip()
|
||||
if len(value) <= int(limit):
|
||||
return value
|
||||
return value[-int(limit):]
|
||||
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
|
||||
|
||||
|
||||
def read_tail_lines(path, max_lines):
|
||||
file_path = Path(path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
return []
|
||||
with file_path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read().splitlines()[-int(max_lines):]
|
||||
|
||||
|
||||
def candidate_domain_roots():
|
||||
roots = []
|
||||
for candidate in (
|
||||
Path.cwd() / "domainCheck",
|
||||
Path("/opt/domaincheck/domainCheck"),
|
||||
Path("/www/wwwroot/getDomain/domainCheck"),
|
||||
):
|
||||
if candidate not in roots:
|
||||
roots.append(candidate)
|
||||
return roots
|
||||
|
||||
|
||||
def candidate_runtime_logs_dirs():
|
||||
dirs = []
|
||||
for candidate in (
|
||||
Path.cwd() / "domain-api" / "runtime" / "logs",
|
||||
Path("/opt/domaincheck/domain-api/runtime/logs"),
|
||||
Path("/www/wwwroot/getDomain/domain-api/runtime/logs"),
|
||||
):
|
||||
if candidate not in dirs:
|
||||
dirs.append(candidate)
|
||||
return dirs
|
||||
|
||||
|
||||
def service_log_file_candidates(service_name):
|
||||
normalized_service_name = str(service_name or "").strip()
|
||||
if not normalized_service_name:
|
||||
return []
|
||||
candidates = []
|
||||
|
||||
def append_candidate(path):
|
||||
if path not in candidates:
|
||||
candidates.append(path)
|
||||
|
||||
if normalized_service_name == (SERVICE_NAMES.get("worker") or "domaincheck-worker"):
|
||||
for domain_root in candidate_domain_roots():
|
||||
append_candidate(domain_root / "detect_worker.log")
|
||||
append_candidate(domain_root / "logs" / "detect_worker.log")
|
||||
elif normalized_service_name == (SERVICE_NAMES.get("api") or "domaincheck-api"):
|
||||
for runtime_logs_dir in candidate_runtime_logs_dirs():
|
||||
append_candidate(runtime_logs_dir / "domain-api.stderr.log")
|
||||
append_candidate(runtime_logs_dir / "domain-api.stdout.log")
|
||||
for domain_root in candidate_domain_roots():
|
||||
append_candidate(domain_root / "logs" / "app.log")
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
def collect_log_file_fallback(service_name, lines):
|
||||
aggregated_lines = []
|
||||
used_paths = []
|
||||
for path in service_log_file_candidates(service_name):
|
||||
current_lines = read_tail_lines(path, lines)
|
||||
if not current_lines:
|
||||
continue
|
||||
aggregated_lines.extend(current_lines)
|
||||
used_paths.append(str(path))
|
||||
if not aggregated_lines:
|
||||
return "", []
|
||||
return "\\n".join(aggregated_lines[-int(lines):]), used_paths
|
||||
|
||||
|
||||
def service_name_from_payload(payload):
|
||||
value = str((payload or {{}}).get("service_name") or "").strip()
|
||||
return value or SERVICE_NAMES.get("worker") or "domaincheck-worker"
|
||||
|
||||
|
||||
def service_name_for_action(action, payload):
|
||||
if action in ("service.status", "service.restart", "service.start", "service.stop"):
|
||||
return service_name_from_payload(payload)
|
||||
runtime_action_map = {{
|
||||
"runtime.start_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
|
||||
"runtime.stop_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
|
||||
"runtime.restart_api": SERVICE_NAMES.get("api") or "domaincheck-api",
|
||||
"runtime.start_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
|
||||
"runtime.stop_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
|
||||
}}
|
||||
return str(runtime_action_map.get(action) or "").strip()
|
||||
|
||||
|
||||
def systemctl_action_name(action):
|
||||
if action in ("service.restart", "runtime.restart_api"):
|
||||
return "restart"
|
||||
if action in ("service.start", "runtime.start_worker", "runtime.start_sync_agent"):
|
||||
return "start"
|
||||
if action in ("service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"):
|
||||
return "stop"
|
||||
return ""
|
||||
|
||||
|
||||
def execute(action, payload):
|
||||
if action == "health.snapshot":
|
||||
checks = {{}}
|
||||
for key, service_name in SERVICE_NAMES.items():
|
||||
code, stdout, stderr = run(["systemctl", "is-active", service_name], timeout=15)
|
||||
checks[key] = {{
|
||||
"service_name": service_name,
|
||||
"returncode": int(code or 0),
|
||||
"state": stdout or stderr,
|
||||
"ok": int(code or 0) == 0 and (stdout or stderr) == "active",
|
||||
}}
|
||||
return True, "health snapshot collected", {{"checks": checks}}
|
||||
|
||||
if action == "service.status":
|
||||
service_name = service_name_from_payload(payload)
|
||||
code, stdout, stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} status collected", result
|
||||
|
||||
systemctl_action = systemctl_action_name(action)
|
||||
if systemctl_action:
|
||||
service_name = service_name_for_action(action, payload)
|
||||
if not service_name:
|
||||
return False, f"missing service name for action: {{action}}", {{"action": action}}
|
||||
code, stdout, stderr = run(["systemctl", systemctl_action, service_name], timeout=45)
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"systemctl_action": systemctl_action,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} {{systemctl_action}} completed", result
|
||||
|
||||
if action == "logs.collect":
|
||||
service_name = service_name_from_payload(payload)
|
||||
lines = max(20, min(int((payload or {{}}).get("lines") or 120), 500))
|
||||
code, stdout, stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
|
||||
journal_output = stdout or stderr
|
||||
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
|
||||
collection_source = "journal"
|
||||
effective_output = journal_output
|
||||
ok = int(code or 0) == 0
|
||||
if (not ok or not stdout.strip()) and fallback_output:
|
||||
collection_source = "file_fallback"
|
||||
effective_output = fallback_output
|
||||
ok = True
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"lines": lines,
|
||||
"collection_source": collection_source,
|
||||
"fallback_used": bool(collection_source == "file_fallback"),
|
||||
"fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
|
||||
"log_paths": fallback_paths,
|
||||
"journal_returncode": int(code or 0),
|
||||
"stdout": trim_output(effective_output, 20000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return ok, effective_output or f"{{service_name}} logs collected", result
|
||||
|
||||
if action == "diagnostics.collect":
|
||||
lines = max(20, min(int((payload or {{}}).get("lines") or 200), 800))
|
||||
diagnostics = {{
|
||||
"services": {{}},
|
||||
"lines": lines,
|
||||
}}
|
||||
for key, service_name in SERVICE_NAMES.items():
|
||||
active_code, active_stdout, active_stderr = run(["systemctl", "is-active", service_name], timeout=15)
|
||||
status_code, status_stdout, status_stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
|
||||
log_code, log_stdout, log_stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
|
||||
log_output = log_stdout or log_stderr
|
||||
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
|
||||
log_source = "journal"
|
||||
if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
|
||||
log_output = fallback_output
|
||||
log_source = "file_fallback"
|
||||
diagnostics["services"][key] = {{
|
||||
"service_name": service_name,
|
||||
"active_returncode": int(active_code or 0),
|
||||
"active_state": active_stdout or active_stderr,
|
||||
"status_returncode": int(status_code or 0),
|
||||
"status_output": trim_output(status_stdout or status_stderr, 12000),
|
||||
"log_returncode": int(log_code or 0),
|
||||
"log_source": log_source,
|
||||
"log_paths": fallback_paths,
|
||||
"log_output": trim_output(log_output, 20000),
|
||||
}}
|
||||
return True, "diagnostics collected", diagnostics
|
||||
|
||||
return False, f"unsupported action: {{action}}", {{"action": action}}
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
ok, message, result = execute(ACTION, PAYLOAD)
|
||||
except Exception as exc:
|
||||
ok, message, result = False, str(exc), {{"exception": str(exc), "action": ACTION}}
|
||||
print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
"""
|
||||
|
||||
|
||||
def _extract_json_from_output(text: str) -> tuple[bool, dict]:
|
||||
for raw_line in reversed(str(text or "").splitlines()):
|
||||
line = str(raw_line or "").strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(data, dict) and "ok" in data:
|
||||
return True, data
|
||||
return False, {}
|
||||
11611
domain-api/app/services/ops_service.py
Normal file
11611
domain-api/app/services/ops_service.py
Normal file
File diff suppressed because it is too large
Load Diff
920
domain-api/app/services/ops_template_service.py
Normal file
920
domain-api/app/services/ops_template_service.py
Normal file
@@ -0,0 +1,920 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_execution_mode_service import decorate_execution_mode_fields, execution_mode_label
|
||||
|
||||
|
||||
def _split_text_list(raw_value: object) -> list[str]:
|
||||
if isinstance(raw_value, list):
|
||||
return [str(item).strip() for item in raw_value if str(item).strip()]
|
||||
text = str(raw_value or "").replace("\r", "\n")
|
||||
if not text.strip():
|
||||
return []
|
||||
normalized = text.replace(",", "\n")
|
||||
return [item.strip() for item in normalized.split("\n") if item.strip()]
|
||||
|
||||
|
||||
def _coerce_bool(raw_value: object) -> bool:
|
||||
if isinstance(raw_value, bool):
|
||||
return raw_value
|
||||
if isinstance(raw_value, (int, float)):
|
||||
return bool(raw_value)
|
||||
normalized = str(raw_value or "").strip().lower()
|
||||
if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
|
||||
return False
|
||||
return bool(normalized)
|
||||
|
||||
|
||||
def _is_missing_value(field_type: str, value: object) -> bool:
|
||||
if field_type == "text_list":
|
||||
return not list(value or [])
|
||||
if field_type == "number":
|
||||
return value is None
|
||||
if field_type == "boolean":
|
||||
return value is None
|
||||
return str(value or "").strip() == ""
|
||||
|
||||
|
||||
def _service_options() -> list[dict]:
|
||||
return [
|
||||
{"label": f"API ({settings.api_service_name})", "value": settings.api_service_name},
|
||||
{"label": f"Worker ({settings.worker_service_name})", "value": settings.worker_service_name},
|
||||
{"label": f"Sync Agent ({settings.sync_agent_service_name})", "value": settings.sync_agent_service_name},
|
||||
{"label": "Node Agent (domaincheck-node-agent)", "value": "domaincheck-node-agent"},
|
||||
]
|
||||
|
||||
|
||||
def _template_catalog() -> list[dict]:
|
||||
service_options = _service_options()
|
||||
return [
|
||||
{
|
||||
"group_key": "runtime",
|
||||
"group_title": "运行时控制",
|
||||
"group_description": "适合日常按钮化控制 Worker / API / Sync Agent。",
|
||||
"items": [
|
||||
{
|
||||
"key": "runtime.start_worker",
|
||||
"title": "启动 Worker",
|
||||
"action": "runtime.start_worker",
|
||||
"description": "启动目标节点的检测 Worker。",
|
||||
"risk_level": "medium",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "runtime.stop_worker",
|
||||
"title": "停止 Worker",
|
||||
"action": "runtime.stop_worker",
|
||||
"description": "停止目标节点的检测 Worker。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "runtime.restart_api",
|
||||
"title": "重启 API",
|
||||
"action": "runtime.restart_api",
|
||||
"description": "重启目标节点的 domaincheck-api 服务。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "runtime.start_sync_agent",
|
||||
"title": "启动 Sync Agent",
|
||||
"action": "runtime.start_sync_agent",
|
||||
"description": "启动目标节点的同步代理服务。",
|
||||
"risk_level": "medium",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "runtime.stop_sync_agent",
|
||||
"title": "停止 Sync Agent",
|
||||
"action": "runtime.stop_sync_agent",
|
||||
"description": "停止目标节点的同步代理服务。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "service.restart",
|
||||
"title": "重启任意服务",
|
||||
"action": "service.restart",
|
||||
"description": "对指定 systemd 服务执行 restart。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "service_name",
|
||||
"label": "服务名",
|
||||
"type": "select",
|
||||
"required": True,
|
||||
"default": settings.worker_service_name,
|
||||
"options": service_options,
|
||||
"help": "默认提供 API / Worker / Sync Agent / Node Agent。",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"group_key": "diagnostics",
|
||||
"group_title": "巡检与取证",
|
||||
"group_description": "适合做联调、排障、日志回收和节点健康诊断。",
|
||||
"items": [
|
||||
{
|
||||
"key": "health.snapshot",
|
||||
"title": "采集健康快照",
|
||||
"action": "health.snapshot",
|
||||
"description": "采集目标节点关键 systemd 服务状态。",
|
||||
"risk_level": "low",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": True,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"key": "logs.collect",
|
||||
"title": "收集服务日志",
|
||||
"action": "logs.collect",
|
||||
"description": "抓取指定服务最近 journalctl 日志。",
|
||||
"risk_level": "low",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": True,
|
||||
"fields": [
|
||||
{
|
||||
"key": "service_name",
|
||||
"label": "服务名",
|
||||
"type": "select",
|
||||
"required": True,
|
||||
"default": settings.worker_service_name,
|
||||
"options": service_options,
|
||||
"help": "默认抓 Worker;排 API 问题时可切到 domaincheck-api。",
|
||||
},
|
||||
{
|
||||
"key": "lines",
|
||||
"label": "日志行数",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 120,
|
||||
"min": 20,
|
||||
"max": 500,
|
||||
"help": "journalctl -n 的行数,适合短时排障。",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "diagnostics.collect",
|
||||
"title": "收集诊断包",
|
||||
"action": "diagnostics.collect",
|
||||
"description": "打包 API / Worker / Sync Agent / Node Agent 的状态与近期日志。",
|
||||
"risk_level": "low",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": True,
|
||||
"fields": [
|
||||
{
|
||||
"key": "lines",
|
||||
"label": "每个服务日志行数",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 200,
|
||||
"min": 20,
|
||||
"max": 800,
|
||||
"help": "越大越适合深度排障,但任务返回会更重。",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "service.status",
|
||||
"title": "采集服务状态",
|
||||
"action": "service.status",
|
||||
"description": "执行 systemctl status --no-pager -l。",
|
||||
"risk_level": "low",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "local-runtime", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": True,
|
||||
"fields": [
|
||||
{
|
||||
"key": "service_name",
|
||||
"label": "服务名",
|
||||
"type": "select",
|
||||
"required": True,
|
||||
"default": settings.worker_service_name,
|
||||
"options": service_options,
|
||||
"help": "用于查看特定服务当前状态和最近日志片段。",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"group_key": "delivery_queue",
|
||||
"group_title": "回执队列治理",
|
||||
"group_description": "适合统一处理 Node Agent 的 pending / dead-letter 回执队列,而不是手工登机删文件。",
|
||||
"items": [
|
||||
{
|
||||
"key": "delivery.queue.flush",
|
||||
"title": "立即冲刷回执队列",
|
||||
"action": "delivery.queue.flush",
|
||||
"description": "立即触发目标节点 Node Agent 冲刷 pending 回执队列。",
|
||||
"risk_level": "low",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": True,
|
||||
"fields": [
|
||||
{
|
||||
"key": "limit",
|
||||
"label": "本次冲刷上限",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 20,
|
||||
"min": 1,
|
||||
"max": 200,
|
||||
"help": "限制本次最多冲刷的回执数量,避免一次返回过大。",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "delivery.queue.replay",
|
||||
"title": "重放死信回执",
|
||||
"action": "delivery.queue.replay",
|
||||
"description": "把 dead-letter 记录重新放回 pending,并可立即触发冲刷。",
|
||||
"risk_level": "medium",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "record_id",
|
||||
"label": "单条记录 ID",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "留空表示按下面的 selector 条件批量匹配。",
|
||||
},
|
||||
{
|
||||
"key": "request_kind",
|
||||
"label": "记录类型",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "可选 job_complete / job_event。",
|
||||
},
|
||||
{
|
||||
"key": "detail_code",
|
||||
"label": "错误码",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "按 last_detail_code 过滤死信记录。",
|
||||
},
|
||||
{
|
||||
"key": "limit",
|
||||
"label": "重放上限",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 20,
|
||||
"min": 1,
|
||||
"max": 200,
|
||||
"help": "批量重放时最多处理多少条死信。",
|
||||
},
|
||||
{
|
||||
"key": "flush_after_replay",
|
||||
"label": "重放后立即冲刷",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "开启后会把重放回 pending 的记录立即补发一次。",
|
||||
},
|
||||
{
|
||||
"key": "reason",
|
||||
"label": "重放原因",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "可选,便于后续回溯为什么执行此次重放。",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "delivery.queue.discard",
|
||||
"title": "丢弃死信回执",
|
||||
"action": "delivery.queue.discard",
|
||||
"description": "把死信移出活动队列,保存到 discarded 归档目录。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "record_id",
|
||||
"label": "单条记录 ID",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "留空表示按下面的 selector 条件批量匹配。",
|
||||
},
|
||||
{
|
||||
"key": "request_kind",
|
||||
"label": "记录类型",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "可选 job_complete / job_event。",
|
||||
},
|
||||
{
|
||||
"key": "detail_code",
|
||||
"label": "错误码",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "按 last_detail_code 过滤死信记录。",
|
||||
},
|
||||
{
|
||||
"key": "limit",
|
||||
"label": "丢弃上限",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 20,
|
||||
"min": 1,
|
||||
"max": 200,
|
||||
"help": "批量丢弃时最多处理多少条死信。",
|
||||
},
|
||||
{
|
||||
"key": "reason",
|
||||
"label": "丢弃原因",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "必须说明为何确认这些死信可以被放弃。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"group_key": "onboarding",
|
||||
"group_title": "接管与纳管",
|
||||
"group_description": "适合在控制面生成节点接入工单、统一沉淀首轮接管动作。",
|
||||
"items": [
|
||||
{
|
||||
"key": "node.bootstrap",
|
||||
"title": "生成节点接入工单",
|
||||
"action": "node.bootstrap",
|
||||
"description": "在控制面签发 Node Agent Token,并生成 bootstrap env / 脚本 / 一键落地命令。",
|
||||
"risk_level": "critical",
|
||||
"default_execution_mode": "control-plane",
|
||||
"execution_modes": ["control-plane"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "control_plane_base_url",
|
||||
"label": "控制面地址",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "可留空。留空时由控制面自动回退到当前 API 地址。",
|
||||
},
|
||||
{
|
||||
"key": "root_dir",
|
||||
"label": "目标安装目录",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "/opt/domaincheck",
|
||||
"help": "bootstrap_node_agent.sh 安装根目录。",
|
||||
},
|
||||
{
|
||||
"key": "expires_in_hours",
|
||||
"label": "Token 有效期(小时)",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 72,
|
||||
"min": 1,
|
||||
"max": 720,
|
||||
"help": "控制新签发 token 的过期时间。",
|
||||
},
|
||||
{
|
||||
"key": "node_region",
|
||||
"label": "节点地域提示",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
|
||||
},
|
||||
{
|
||||
"key": "node_role",
|
||||
"label": "节点角色提示",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"help": "可留空;留空时优先从托管节点或集群快照自动推断。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"group_key": "release",
|
||||
"group_title": "发布与变更",
|
||||
"group_description": "适合把单节点灰度、点状修复和手工发布收编到标准 deploy.release 任务。",
|
||||
"items": [
|
||||
{
|
||||
"key": "deploy.release.control",
|
||||
"title": "发布控制面节点",
|
||||
"action": "deploy.release",
|
||||
"description": "面向 controller 节点的标准发布任务,默认会重启 API / Worker / Sync Agent 并执行 API 健康检查。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "ssh"],
|
||||
"target_roles": ["control"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "release_version",
|
||||
"label": "版本号",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"help": "例如 2026.04.18-rc1,用于 releases/<version> 目录名。",
|
||||
},
|
||||
{
|
||||
"key": "artifact_url",
|
||||
"label": "发布包地址",
|
||||
"type": "textarea",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
|
||||
},
|
||||
{
|
||||
"key": "checksum",
|
||||
"label": "SHA256 校验",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "可留空;填写后会强校验发布包完整性。",
|
||||
},
|
||||
{
|
||||
"key": "install_root",
|
||||
"label": "安装根目录",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "/opt/domaincheck",
|
||||
"help": "目标节点上的 releases/current 根目录。",
|
||||
},
|
||||
{
|
||||
"key": "restart_services",
|
||||
"label": "重启服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [
|
||||
settings.api_service_name,
|
||||
settings.worker_service_name,
|
||||
settings.sync_agent_service_name,
|
||||
],
|
||||
"wide": True,
|
||||
"rows": 3,
|
||||
"help": "每行一个服务。控制面默认重启 API / Worker / Sync Agent。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_urls",
|
||||
"label": "健康检查 URL",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": ["http://127.0.0.1:8100/health"],
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "每行一个 URL。控制面默认探活本机 API /health。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_services",
|
||||
"label": "健康检查服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [
|
||||
settings.api_service_name,
|
||||
settings.worker_service_name,
|
||||
settings.sync_agent_service_name,
|
||||
],
|
||||
"wide": True,
|
||||
"rows": 3,
|
||||
"help": "每行一个 systemd 服务,发布后会检查其 active 状态。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_timeout_seconds",
|
||||
"label": "URL 超时",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 10,
|
||||
"min": 2,
|
||||
"max": 120,
|
||||
"help": "每次 URL 探活超时秒数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_retries",
|
||||
"label": "重试次数",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 10,
|
||||
"help": "健康检查失败后的额外重试次数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_interval_seconds",
|
||||
"label": "重试间隔",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 60,
|
||||
"help": "每次重试前的等待秒数。",
|
||||
},
|
||||
{
|
||||
"key": "rollback_on_failure",
|
||||
"label": "失败自动回滚",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "健康检查失败时自动切回旧 current 并重启服务。",
|
||||
},
|
||||
{
|
||||
"key": "switch_current",
|
||||
"label": "切换 current 软链",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "关闭后只解压版本目录,不切 current。",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "deploy.release.worker",
|
||||
"title": "发布 Worker 节点",
|
||||
"action": "deploy.release",
|
||||
"description": "面向独立 Worker 节点的标准发布任务,默认只重启 Worker 并校验 Worker 服务状态。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "ssh"],
|
||||
"target_roles": ["worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "release_version",
|
||||
"label": "版本号",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"help": "例如 2026.04.18-rc1,用于 releases/<version> 目录名。",
|
||||
},
|
||||
{
|
||||
"key": "artifact_url",
|
||||
"label": "发布包地址",
|
||||
"type": "textarea",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
|
||||
},
|
||||
{
|
||||
"key": "checksum",
|
||||
"label": "SHA256 校验",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "可留空;填写后会强校验发布包完整性。",
|
||||
},
|
||||
{
|
||||
"key": "install_root",
|
||||
"label": "安装根目录",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "/opt/domaincheck",
|
||||
"help": "目标节点上的 releases/current 根目录。",
|
||||
},
|
||||
{
|
||||
"key": "restart_services",
|
||||
"label": "重启服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [settings.worker_service_name],
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "每行一个服务。独立 Worker 默认只重启 Worker。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_urls",
|
||||
"label": "健康检查 URL",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [],
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "Worker 节点通常可留空;为空时不做 URL 探活。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_services",
|
||||
"label": "健康检查服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [settings.worker_service_name],
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "每行一个 systemd 服务,默认只校验 Worker active 状态。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_timeout_seconds",
|
||||
"label": "URL 超时",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 10,
|
||||
"min": 2,
|
||||
"max": 120,
|
||||
"help": "每次 URL 探活超时秒数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_retries",
|
||||
"label": "重试次数",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 10,
|
||||
"help": "健康检查失败后的额外重试次数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_interval_seconds",
|
||||
"label": "重试间隔",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 60,
|
||||
"help": "每次重试前的等待秒数。",
|
||||
},
|
||||
{
|
||||
"key": "rollback_on_failure",
|
||||
"label": "失败自动回滚",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "健康检查失败时自动切回旧 current 并重启服务。",
|
||||
},
|
||||
{
|
||||
"key": "switch_current",
|
||||
"label": "切换 current 软链",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "关闭后只解压版本目录,不切 current。",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"key": "deploy.release.custom",
|
||||
"title": "发布自定义节点",
|
||||
"action": "deploy.release",
|
||||
"description": "完全自定义 deploy.release 参数,适合灰度验证、特殊节点或后续扩展场景。",
|
||||
"risk_level": "high",
|
||||
"default_execution_mode": "remote-agent",
|
||||
"execution_modes": ["remote-agent", "ssh"],
|
||||
"target_roles": ["control", "worker"],
|
||||
"default_auto_approve": False,
|
||||
"fields": [
|
||||
{
|
||||
"key": "release_version",
|
||||
"label": "版本号",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"help": "例如 2026.04.18-rc1,用于 releases/<version> 目录名。",
|
||||
},
|
||||
{
|
||||
"key": "artifact_url",
|
||||
"label": "发布包地址",
|
||||
"type": "textarea",
|
||||
"required": True,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
|
||||
},
|
||||
{
|
||||
"key": "checksum",
|
||||
"label": "SHA256 校验",
|
||||
"type": "text",
|
||||
"required": False,
|
||||
"default": "",
|
||||
"wide": True,
|
||||
"help": "可留空;填写后会强校验发布包完整性。",
|
||||
},
|
||||
{
|
||||
"key": "install_root",
|
||||
"label": "安装根目录",
|
||||
"type": "text",
|
||||
"required": True,
|
||||
"default": "/opt/domaincheck",
|
||||
"help": "目标节点上的 releases/current 根目录。",
|
||||
},
|
||||
{
|
||||
"key": "restart_services",
|
||||
"label": "重启服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [],
|
||||
"wide": True,
|
||||
"rows": 3,
|
||||
"help": "每行一个服务。留空表示只落盘版本,不主动重启任何服务。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_urls",
|
||||
"label": "健康检查 URL",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [],
|
||||
"wide": True,
|
||||
"rows": 2,
|
||||
"help": "每行一个 URL。留空表示不做 URL 探活。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_services",
|
||||
"label": "健康检查服务",
|
||||
"type": "text_list",
|
||||
"required": False,
|
||||
"default": [],
|
||||
"wide": True,
|
||||
"rows": 3,
|
||||
"help": "每行一个 systemd 服务。显式留空时不会自动回落到默认 API 检查。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_timeout_seconds",
|
||||
"label": "URL 超时",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 10,
|
||||
"min": 2,
|
||||
"max": 120,
|
||||
"help": "每次 URL 探活超时秒数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_retries",
|
||||
"label": "重试次数",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 10,
|
||||
"help": "健康检查失败后的额外重试次数。",
|
||||
},
|
||||
{
|
||||
"key": "health_check_interval_seconds",
|
||||
"label": "重试间隔",
|
||||
"type": "number",
|
||||
"required": True,
|
||||
"default": 2,
|
||||
"min": 0,
|
||||
"max": 60,
|
||||
"help": "每次重试前的等待秒数。",
|
||||
},
|
||||
{
|
||||
"key": "rollback_on_failure",
|
||||
"label": "失败自动回滚",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "健康检查失败时自动切回旧 current 并重启服务。",
|
||||
},
|
||||
{
|
||||
"key": "switch_current",
|
||||
"label": "切换 current 软链",
|
||||
"type": "boolean",
|
||||
"required": True,
|
||||
"default": True,
|
||||
"help": "关闭后只解压版本目录,不切 current。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def get_ops_action_templates() -> dict:
|
||||
groups = deepcopy(_template_catalog())
|
||||
items: list[dict] = []
|
||||
for group in groups:
|
||||
group_key = str(group.get("group_key") or "").strip()
|
||||
group_title = str(group.get("group_title") or "").strip()
|
||||
group_description = str(group.get("group_description") or "").strip()
|
||||
normalized_group_items: list[dict] = []
|
||||
for item in list(group.get("items") or []):
|
||||
normalized_item = decorate_execution_mode_fields(dict(item))
|
||||
normalized_item["group_key"] = group_key
|
||||
normalized_item["group_title"] = group_title
|
||||
normalized_item["group_description"] = group_description
|
||||
normalized_group_items.append(normalized_item)
|
||||
items.append(normalized_item)
|
||||
group["items"] = normalized_group_items
|
||||
return {
|
||||
"groups": groups,
|
||||
"items": items,
|
||||
"defaults": {
|
||||
"requested_by": "web-ui",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": execution_mode_label("remote-agent"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_ops_action_template(template_key: str) -> dict:
|
||||
normalized_key = str(template_key or "").strip()
|
||||
if not normalized_key:
|
||||
return {}
|
||||
for group in _template_catalog():
|
||||
group_key = str(group.get("group_key") or "").strip()
|
||||
group_title = str(group.get("group_title") or "").strip()
|
||||
group_description = str(group.get("group_description") or "").strip()
|
||||
for item in list(group.get("items") or []):
|
||||
if str(item.get("key") or "").strip() == normalized_key:
|
||||
normalized_item = decorate_execution_mode_fields(deepcopy(item))
|
||||
normalized_item["group_key"] = group_key
|
||||
normalized_item["group_title"] = group_title
|
||||
normalized_item["group_description"] = group_description
|
||||
return normalized_item
|
||||
return {}
|
||||
|
||||
|
||||
def build_ops_template_payload(template_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
template = get_ops_action_template(template_key)
|
||||
if not template:
|
||||
return False, "动作模板不存在", {}
|
||||
|
||||
source_payload = dict(payload or {})
|
||||
normalized_payload: dict = {}
|
||||
for field in list(template.get("fields") or []):
|
||||
field_key = str(field.get("key") or "").strip()
|
||||
if not field_key:
|
||||
continue
|
||||
field_type = str(field.get("type") or "text").strip()
|
||||
raw_value = source_payload.get(field_key, field.get("default"))
|
||||
if field_type == "number":
|
||||
try:
|
||||
value = int(raw_value or 0)
|
||||
except Exception:
|
||||
return False, f"{field_key} 必须是数字", {}
|
||||
min_value = field.get("min")
|
||||
max_value = field.get("max")
|
||||
if min_value is not None:
|
||||
value = max(int(min_value), value)
|
||||
if max_value is not None:
|
||||
value = min(int(max_value), value)
|
||||
elif field_type == "boolean":
|
||||
value = _coerce_bool(raw_value)
|
||||
elif field_type == "text_list":
|
||||
value = _split_text_list(raw_value)
|
||||
else:
|
||||
value = str(raw_value or "").strip()
|
||||
if bool(field.get("required", False)) and _is_missing_value(field_type, value):
|
||||
return False, f"{field_key} 不能为空", {}
|
||||
normalized_payload[field_key] = value
|
||||
return True, "ok", normalized_payload
|
||||
@@ -7,6 +7,7 @@ from app.core.config import settings
|
||||
from app.core.db import get_db
|
||||
from app.core.files import read_json
|
||||
from app.core.redis_client import get_redis
|
||||
from app.services.build_info_service import get_runtime_build_info
|
||||
from app.services.cluster_runtime_service import get_cluster_snapshot
|
||||
from app.services.detect_service import get_detect_status
|
||||
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
|
||||
@@ -187,6 +188,89 @@ def _build_multi_region_readiness(
|
||||
}
|
||||
|
||||
|
||||
def _detect_participation_snapshot(*, row: dict) -> dict:
|
||||
items_running = int(row.get("items_running", 0) or 0)
|
||||
items_claimed = int(row.get("items_claimed", 0) or 0)
|
||||
processed_recent = int(row.get("processed_recent", 0) or 0)
|
||||
current_load = int(row.get("current_load", 0) or 0)
|
||||
status = str(row.get("status") or "").strip().lower()
|
||||
|
||||
if items_running > 0:
|
||||
return {
|
||||
"participation_state": "running",
|
||||
"participation_label": "执行中",
|
||||
"participation_reason": f"当前正在执行 {items_running} 项检测任务。",
|
||||
"is_current_participant": True,
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
if items_claimed > 0:
|
||||
return {
|
||||
"participation_state": "claimed",
|
||||
"participation_label": "已领待跑",
|
||||
"participation_reason": f"已领取 {items_claimed} 项任务,等待线程继续执行。",
|
||||
"is_current_participant": True,
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
if processed_recent > 0:
|
||||
return {
|
||||
"participation_state": "recent_throughput",
|
||||
"participation_label": "近窗有吞吐",
|
||||
"participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。",
|
||||
"is_current_participant": True,
|
||||
"is_dispatch_active": False,
|
||||
}
|
||||
if current_load > 0 or status == "busy":
|
||||
load_value = max(current_load, 1)
|
||||
return {
|
||||
"participation_state": "load_syncing",
|
||||
"participation_label": "负载待确认",
|
||||
"participation_reason": f"节点当前负载为 {load_value},但还未观察到已领、执行中或近窗吞吐数据,先归入在线未参与观察。",
|
||||
"is_current_participant": False,
|
||||
"is_dispatch_active": False,
|
||||
}
|
||||
return {
|
||||
"participation_state": "standby",
|
||||
"participation_label": "在线待命",
|
||||
"participation_reason": "当前未领任务、未执行任务,也没有近窗吞吐。",
|
||||
"is_current_participant": False,
|
||||
"is_dispatch_active": False,
|
||||
}
|
||||
|
||||
|
||||
def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict:
|
||||
metadata = cluster_node.get("metadata") or {}
|
||||
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
|
||||
region = str(cluster_node.get("region") or settings.node_region)
|
||||
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
|
||||
items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
|
||||
items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
|
||||
items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
|
||||
items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
|
||||
items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
|
||||
row = {
|
||||
"node_code": node_code,
|
||||
"role": role,
|
||||
"region": region,
|
||||
"status": str(cluster_node.get("status") or "unknown"),
|
||||
"is_effective_worker": is_effective_worker,
|
||||
"detect_participating": False,
|
||||
"current_load": int(cluster_node.get("current_load", 0) or 0),
|
||||
"items_total": items_total,
|
||||
"items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
|
||||
"items_claimed": items_claimed,
|
||||
"items_running": items_running,
|
||||
"items_completed": items_completed,
|
||||
"items_failed": items_failed,
|
||||
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
|
||||
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
|
||||
"metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"),
|
||||
}
|
||||
row.update(_detect_participation_snapshot(row=row))
|
||||
row["detect_participating"] = bool(row.get("is_current_participant", False))
|
||||
return row
|
||||
|
||||
|
||||
def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]:
|
||||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||||
active_job = detect_snapshot.get("active_job") or {}
|
||||
@@ -204,56 +288,25 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
|
||||
}
|
||||
|
||||
merged: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for item in list(active_job.get("node_stats") or []):
|
||||
node_code = str(item.get("node_code") or "").strip()
|
||||
if not node_code or node_code == "unassigned":
|
||||
continue
|
||||
seen.add(node_code)
|
||||
job_map = {
|
||||
str(item.get("node_code") or "").strip(): item
|
||||
for item in list(active_job.get("node_stats") or [])
|
||||
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
|
||||
}
|
||||
candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map))
|
||||
for node_code in candidate_codes:
|
||||
cluster_node = cluster_map.get(node_code, {})
|
||||
queue_node = queue_map.get(node_code, {})
|
||||
merged.append(
|
||||
{
|
||||
"node_code": node_code,
|
||||
"role": str(cluster_node.get("role") or "worker"),
|
||||
"region": str(cluster_node.get("region") or settings.node_region),
|
||||
"status": str(cluster_node.get("status") or "unknown"),
|
||||
"is_effective_worker": bool(cluster_node.get("is_effective_worker", False) or str(cluster_node.get("role") or "") == "worker"),
|
||||
"detect_participating": True,
|
||||
"current_load": int(cluster_node.get("current_load", 0) or 0),
|
||||
"items_total": int(item.get("items_total", 0) or 0),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"items_completed": int(item.get("items_completed", 0) or 0),
|
||||
"items_failed": int(item.get("items_failed", 0) or 0),
|
||||
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
|
||||
}
|
||||
if cluster_node and not bool(cluster_node.get("is_effective_worker", False)):
|
||||
continue
|
||||
row = _build_detect_node_row(
|
||||
node_code=node_code,
|
||||
cluster_node=cluster_node,
|
||||
job_node=job_map.get(node_code, {}),
|
||||
queue_node=queue_map.get(node_code, {}),
|
||||
)
|
||||
|
||||
if worker_runtime.get("running", False) and settings.node_code not in seen:
|
||||
cluster_node = cluster_map.get(settings.node_code, {})
|
||||
if cluster_node:
|
||||
merged.append(
|
||||
{
|
||||
"node_code": settings.node_code,
|
||||
"role": str(cluster_node.get("role") or settings.node_role),
|
||||
"region": str(cluster_node.get("region") or settings.node_region),
|
||||
"status": str(cluster_node.get("status") or "online"),
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": True,
|
||||
"current_load": int(cluster_node.get("current_load", 0) or 0),
|
||||
"items_total": 0,
|
||||
"items_pending": 0,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"items_completed": 0,
|
||||
"items_failed": 0,
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0,
|
||||
}
|
||||
)
|
||||
if not bool(row.get("is_current_participant", False)):
|
||||
continue
|
||||
merged.append(row)
|
||||
|
||||
return sorted(
|
||||
merged,
|
||||
@@ -261,11 +314,143 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
|
||||
-int(item.get("items_running", 0) or 0),
|
||||
-int(item.get("items_claimed", 0) or 0),
|
||||
-int(item.get("processed_recent", 0) or 0),
|
||||
-int(item.get("current_load", 0) or 0),
|
||||
str(item.get("node_code") or ""),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]:
|
||||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||||
participating_codes = {
|
||||
str(item.get("node_code") or "").strip()
|
||||
for item in list(participating_nodes or [])
|
||||
if str(item.get("node_code") or "").strip()
|
||||
}
|
||||
standby_rows: list[dict] = []
|
||||
for node in cluster_nodes:
|
||||
node_code = str(node.get("node_code") or "").strip()
|
||||
node_status = str(node.get("status") or "").strip()
|
||||
node_current_load = int(node.get("current_load", 0) or 0)
|
||||
metadata = node.get("metadata") or {}
|
||||
if not node_code:
|
||||
continue
|
||||
if not bool(node.get("is_effective_worker", False)):
|
||||
continue
|
||||
if node_status not in {"online", "busy"}:
|
||||
continue
|
||||
if node_code in participating_codes:
|
||||
continue
|
||||
standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby"
|
||||
standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命"
|
||||
standby_reason = (
|
||||
str(metadata.get("detail") or "").strip()
|
||||
or str(metadata.get("phase_detail") or "").strip()
|
||||
or (
|
||||
f"当前阶段:{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}"
|
||||
if str(metadata.get("phase") or metadata.get("phase_label") or "").strip()
|
||||
else ""
|
||||
)
|
||||
or (
|
||||
f"节点当前负载为 {node_current_load},但还未观察到已领、执行中或近窗吞吐数据。"
|
||||
if standby_state == "load_syncing"
|
||||
else "节点在线,当前未领任务、未执行任务,也没有近窗吞吐。"
|
||||
)
|
||||
)
|
||||
standby_rows.append(
|
||||
{
|
||||
"node_code": node_code,
|
||||
"role": str(node.get("role") or "worker"),
|
||||
"region": str(node.get("region") or settings.node_region),
|
||||
"status": node_status,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
"participation_state": standby_state,
|
||||
"participation_label": standby_label,
|
||||
"participation_reason": standby_reason,
|
||||
"current_load": node_current_load,
|
||||
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
|
||||
"phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
|
||||
"detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
|
||||
"worker_online": bool(metadata.get("worker_online", False)),
|
||||
"standby_reason": standby_reason,
|
||||
}
|
||||
)
|
||||
return sorted(
|
||||
standby_rows,
|
||||
key=lambda item: (
|
||||
str(item.get("status") or ""),
|
||||
str(item.get("role") or ""),
|
||||
str(item.get("node_code") or ""),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_detect_participation_summary(
|
||||
*,
|
||||
participating_nodes: list[dict],
|
||||
standby_nodes: list[dict],
|
||||
cluster_snapshot: dict,
|
||||
) -> dict:
|
||||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||||
effective_online_nodes = [
|
||||
node
|
||||
for node in cluster_nodes
|
||||
if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"}
|
||||
]
|
||||
dispatch_active_nodes = [
|
||||
row for row in participating_nodes
|
||||
if bool(row.get("is_dispatch_active", False))
|
||||
]
|
||||
recent_only_nodes = [
|
||||
row for row in participating_nodes
|
||||
if str(row.get("participation_state") or "").strip() == "recent_throughput"
|
||||
]
|
||||
load_syncing_nodes = [
|
||||
row for row in standby_nodes
|
||||
if str(row.get("participation_state") or "").strip() == "load_syncing"
|
||||
]
|
||||
pure_standby_nodes = [
|
||||
row for row in standby_nodes
|
||||
if str(row.get("participation_state") or "").strip() == "standby"
|
||||
]
|
||||
dedicated_worker_nodes = [
|
||||
node for node in effective_online_nodes
|
||||
if str(node.get("role") or "").strip() == "worker"
|
||||
]
|
||||
controller_worker_nodes = [
|
||||
node for node in effective_online_nodes
|
||||
if str(node.get("role") or "").strip() == "control"
|
||||
]
|
||||
|
||||
summary_parts = [
|
||||
f"有效执行节点 {len(effective_online_nodes)} 台",
|
||||
f"正在执行/领任务 {len(dispatch_active_nodes)} 台",
|
||||
f"近窗刚有吞吐 {len(recent_only_nodes)} 台",
|
||||
f"在线但未参与 {len(standby_nodes)} 台",
|
||||
]
|
||||
if load_syncing_nodes:
|
||||
summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台")
|
||||
|
||||
return {
|
||||
"effective_online_nodes": len(effective_online_nodes),
|
||||
"participating_nodes": len(participating_nodes),
|
||||
"dispatch_active_nodes": len(dispatch_active_nodes),
|
||||
"recent_only_nodes": len(recent_only_nodes),
|
||||
"non_participating_nodes": len(standby_nodes),
|
||||
"standby_nodes": len(pure_standby_nodes),
|
||||
"load_syncing_nodes": len(load_syncing_nodes),
|
||||
"dedicated_worker_nodes": len(dedicated_worker_nodes),
|
||||
"controller_worker_nodes": len(controller_worker_nodes),
|
||||
"dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes],
|
||||
"recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes],
|
||||
"non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes],
|
||||
"standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes],
|
||||
"load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes],
|
||||
"summary": ";".join(summary_parts),
|
||||
}
|
||||
|
||||
|
||||
def get_runtime_status() -> dict:
|
||||
runtime_settings = get_runtime_settings()
|
||||
worker_runtime = detect_worker_runtime()
|
||||
@@ -314,6 +499,17 @@ def get_runtime_status() -> dict:
|
||||
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
|
||||
"queue_health": queue_health,
|
||||
"capacity_plan": capacity_plan,
|
||||
"log_sync": {
|
||||
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
|
||||
"mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"),
|
||||
"line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0),
|
||||
"source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0),
|
||||
"source_nodes": list(detect_snapshot.get("remote_log_nodes") or []),
|
||||
"source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []),
|
||||
"last_at": str(detect_snapshot.get("remote_log_last_at") or ""),
|
||||
"last_line": str(detect_snapshot.get("remote_log_last_line") or ""),
|
||||
"preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:],
|
||||
},
|
||||
}
|
||||
if not worker_expected_on_this_node:
|
||||
detect_payload.update(
|
||||
@@ -349,6 +545,16 @@ def get_runtime_status() -> dict:
|
||||
detect_snapshot=detect_payload,
|
||||
worker_runtime=worker_runtime,
|
||||
)
|
||||
detect_payload["standby_nodes"] = _build_standby_detect_nodes(
|
||||
cluster_snapshot=cluster_snapshot,
|
||||
participating_nodes=detect_payload["participating_nodes"],
|
||||
)
|
||||
detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or [])
|
||||
detect_payload["participation_summary"] = _build_detect_participation_summary(
|
||||
participating_nodes=detect_payload["participating_nodes"],
|
||||
standby_nodes=detect_payload["non_participating_nodes"],
|
||||
cluster_snapshot=cluster_snapshot,
|
||||
)
|
||||
append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
|
||||
sync_summary = get_sync_summary(record_limit=5)
|
||||
readiness = _build_multi_region_readiness(
|
||||
@@ -357,6 +563,7 @@ def get_runtime_status() -> dict:
|
||||
worker_runtime=worker_runtime,
|
||||
sync_agent_runtime=sync_agent_runtime,
|
||||
)
|
||||
build_info = get_runtime_build_info()
|
||||
|
||||
return {
|
||||
"api": {
|
||||
@@ -371,6 +578,7 @@ def get_runtime_status() -> dict:
|
||||
"health_url": f"http://127.0.0.1:{settings.api_port}/health",
|
||||
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
|
||||
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
|
||||
"build": build_info,
|
||||
},
|
||||
"node": {
|
||||
"code": settings.node_code,
|
||||
|
||||
@@ -450,6 +450,17 @@ def append_runtime_projection_if_changed(
|
||||
continue
|
||||
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
|
||||
break
|
||||
local_job_bucket = {}
|
||||
for item in list(active_job.get("node_stats") or []):
|
||||
if str(item.get("node_code") or "").strip() != settings.node_code:
|
||||
continue
|
||||
local_job_bucket = item
|
||||
break
|
||||
if not local_participating:
|
||||
local_participating = bool(
|
||||
int(local_job_bucket.get("items_running", 0) or 0) > 0
|
||||
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
|
||||
)
|
||||
projection = {
|
||||
"node": {
|
||||
"node_code": settings.node_code,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,7 +12,7 @@ TEMPLATE_DIR="$API_DIR/deploy/multi-region/templates"
|
||||
|
||||
if [ "$ROLE" != "controller" ] && [ "$ROLE" != "worker" ]; then
|
||||
echo "invalid role: $ROLE"
|
||||
echo "usage: bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]"
|
||||
echo "usage: bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
61
domain-api/deploy/multi-region/bootstrap_node_agent.sh
Executable file
61
domain-api/deploy/multi-region/bootstrap_node_agent.sh
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="${1:-/opt/domaincheck}"
|
||||
ENV_FILE="/etc/default/domaincheck-node-agent"
|
||||
SERVICE_FILE="/etc/systemd/system/domaincheck-node-agent.service"
|
||||
SERVICE_NAME="domaincheck-node-agent"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
resolve_project_root() {
|
||||
local candidate
|
||||
for candidate in \
|
||||
"${ROOT_DIR}/domainCheck" \
|
||||
"${ROOT_DIR}/domain-api" \
|
||||
"${ROOT_DIR}" \
|
||||
"$(cd "${SCRIPT_DIR}/../.." && pwd)"; do
|
||||
if [[ -f "${candidate}/deploy/systemd/domain-node-agent.service" && -f "${candidate}/deploy/multi-region/templates/domaincheck-node-agent.env.example" ]]; then
|
||||
printf '%s' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
PROJECT_ROOT="$(resolve_project_root || true)"
|
||||
if [[ -z "${PROJECT_ROOT}" ]]; then
|
||||
echo "unable to resolve project root under ${ROOT_DIR} or ${SCRIPT_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TEMPLATE_FILE="${PROJECT_ROOT}/deploy/multi-region/templates/domaincheck-node-agent.env.example"
|
||||
UNIT_TEMPLATE="${PROJECT_ROOT}/deploy/systemd/domain-node-agent.service"
|
||||
|
||||
echo "[1/5] install systemd unit"
|
||||
install -m 0644 "${UNIT_TEMPLATE}" "${SERVICE_FILE}"
|
||||
|
||||
echo "[2/5] install env template if absent"
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
install -m 0644 "${TEMPLATE_FILE}" "${ENV_FILE}"
|
||||
echo "created ${ENV_FILE}"
|
||||
else
|
||||
echo "keep existing ${ENV_FILE}"
|
||||
fi
|
||||
|
||||
echo "[3/5] reload systemd"
|
||||
systemctl daemon-reload
|
||||
|
||||
echo "[4/5] enable service"
|
||||
systemctl enable "${SERVICE_NAME}" >/dev/null
|
||||
|
||||
echo "[5/5] next steps"
|
||||
echo "project_root=${PROJECT_ROOT}"
|
||||
echo "edit ${ENV_FILE} and set:"
|
||||
echo " OPS_CONTROL_PLANE_BASE_URL"
|
||||
echo " OPS_AGENT_TOKEN"
|
||||
echo " NODE_CODE / NODE_REGION / NODE_ROLE"
|
||||
echo
|
||||
echo "then run:"
|
||||
echo " systemctl restart ${SERVICE_NAME}"
|
||||
echo " systemctl status ${SERVICE_NAME} --no-pager -l"
|
||||
91
domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh
Executable file
91
domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
API_BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
NODE_CODE="${2:-}"
|
||||
NODE_REGION="${3:-mainland}"
|
||||
NODE_ROLE="${4:-worker}"
|
||||
CONTROL_PLANE_BASE_URL="${5:-${API_BASE_URL}}"
|
||||
ROOT_DIR="${6:-/opt/domaincheck}"
|
||||
ISSUED_BY="${ISSUED_BY:-cli-bootstrap}"
|
||||
EXPIRES_IN_HOURS="${EXPIRES_IN_HOURS:-72}"
|
||||
|
||||
if [[ -z "${NODE_CODE}" ]]; then
|
||||
echo "usage: $0 <api_base_url> <node_code> [node_region] [node_role] [control_plane_base_url] [root_dir]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NORMALIZED_API_BASE_URL="${API_BASE_URL%/}"
|
||||
NORMALIZED_CONTROL_PLANE_BASE_URL="${CONTROL_PLANE_BASE_URL%/}"
|
||||
if [[ "${NORMALIZED_API_BASE_URL}" != */api/v1 ]]; then
|
||||
API_ENDPOINT="${NORMALIZED_API_BASE_URL}/api/v1/ops/agent/bootstrap-plan"
|
||||
else
|
||||
API_ENDPOINT="${NORMALIZED_API_BASE_URL}/ops/agent/bootstrap-plan"
|
||||
fi
|
||||
|
||||
TMP_RESPONSE_FILE="$(mktemp)"
|
||||
cleanup() {
|
||||
rm -f "${TMP_RESPONSE_FILE}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
python3 - "${NODE_CODE}" "${NODE_REGION}" "${NODE_ROLE}" "${ISSUED_BY}" "${EXPIRES_IN_HOURS}" "${NORMALIZED_CONTROL_PLANE_BASE_URL}" "${ROOT_DIR}" >"${TMP_RESPONSE_FILE}.payload" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = {
|
||||
"node_code": sys.argv[1],
|
||||
"node_region": sys.argv[2],
|
||||
"node_role": sys.argv[3],
|
||||
"issued_by": sys.argv[4],
|
||||
"expires_in_hours": int(sys.argv[5]),
|
||||
"control_plane_base_url": sys.argv[6],
|
||||
"root_dir": sys.argv[7],
|
||||
"metadata": {
|
||||
"issued_from": "build_node_agent_bootstrap_plan.sh",
|
||||
},
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
|
||||
curl -fsS -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data-binary @"${TMP_RESPONSE_FILE}.payload" \
|
||||
"${API_ENDPOINT}" >"${TMP_RESPONSE_FILE}"
|
||||
|
||||
python3 - "${TMP_RESPONSE_FILE}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
response = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||
response_code = response.get("code", 0)
|
||||
if int(response_code if response_code is not None else 0) != 0:
|
||||
raise SystemExit(response.get("message") or "bootstrap plan request failed")
|
||||
|
||||
data = response.get("data") or {}
|
||||
plan = data.get("bootstrap_plan") or {}
|
||||
|
||||
print("[token]")
|
||||
print(f"node_code={data.get('node_code', '')}")
|
||||
print(f"token_preview={data.get('token_preview', '')}")
|
||||
print(f"expires_at={data.get('expires_at', '')}")
|
||||
print()
|
||||
print("[env]")
|
||||
print(plan.get("env_content") or "")
|
||||
print()
|
||||
print("[commands]")
|
||||
print(plan.get("command_block") or "")
|
||||
print()
|
||||
print("[bootstrap-script]")
|
||||
print(plan.get("bootstrap_script_content") or "")
|
||||
print()
|
||||
print("[bootstrap-run-block]")
|
||||
print(plan.get("bootstrap_run_script_block") or "")
|
||||
print()
|
||||
print("[health-checks]")
|
||||
for item in plan.get("health_checks") or []:
|
||||
print(item)
|
||||
PY
|
||||
|
||||
rm -f "${TMP_RESPONSE_FILE}.payload"
|
||||
345
domain-api/deploy/multi-region/check_env_drift.sh
Normal file
345
domain-api/deploy/multi-region/check_env_drift.sh
Normal file
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
API_SERVICE="${API_SERVICE:-domaincheck-api}"
|
||||
WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}"
|
||||
SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}"
|
||||
NODE_AGENT_SERVICE="${NODE_AGENT_SERVICE:-domaincheck-node-agent}"
|
||||
API_ENV_FILE="${API_ENV_FILE:-/etc/default/domaincheck-api}"
|
||||
WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/default/domaincheck-worker}"
|
||||
NODE_AGENT_ENV_FILE="${NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
API_DIR="${PROJECT_ROOT}/domain-api"
|
||||
WEB_DIR="${PROJECT_ROOT}/domain-web"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
|
||||
FETCH_BODY=""
|
||||
FETCH_STATUS="000"
|
||||
|
||||
fetch_json() {
|
||||
local url="${1:-}"
|
||||
local raw_text
|
||||
raw_text="$(curl -sS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT:-3}" \
|
||||
--max-time "${CURL_MAX_TIME:-10}" \
|
||||
"${url}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
||||
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
|
||||
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
|
||||
}
|
||||
|
||||
echo "[1/8] local command availability"
|
||||
for cmd in bash curl systemctl "${PYTHON_BIN}" node npm; do
|
||||
if command -v "${cmd}" >/dev/null 2>&1; then
|
||||
printf '%s: ok -> %s\n' "${cmd}" "$(command -v "${cmd}")"
|
||||
else
|
||||
printf '%s: missing\n' "${cmd}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[2/8] python environment"
|
||||
"${PYTHON_BIN}" - <<'PY' "${API_DIR}" "${WEB_DIR}"
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
api_dir = pathlib.Path(sys.argv[1])
|
||||
web_dir = pathlib.Path(sys.argv[2])
|
||||
|
||||
def has_module(name: str) -> bool:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
|
||||
payload = {
|
||||
"python_executable": sys.executable,
|
||||
"python_version": sys.version.split()[0],
|
||||
"pytest_installed": has_module("pytest"),
|
||||
"uvicorn_installed": has_module("uvicorn"),
|
||||
"fastapi_installed": has_module("fastapi"),
|
||||
"api_requirements_exists": (api_dir / "requirements.txt").exists(),
|
||||
"web_package_json_exists": (web_dir / "package.json").exists(),
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[3/8] env files"
|
||||
for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${NODE_AGENT_ENV_FILE}"; do
|
||||
if [[ -f "${env_file}" ]]; then
|
||||
echo "exists: ${env_file}"
|
||||
else
|
||||
echo "missing: ${env_file}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[4/8] systemd service states"
|
||||
for service in "${API_SERVICE}" "${WORKER_SERVICE}" "${SYNC_AGENT_SERVICE}" "${NODE_AGENT_SERVICE}"; do
|
||||
if systemctl list-unit-files "${service}.service" --no-pager >/dev/null 2>&1; then
|
||||
printf '%s: enabled=%s active=%s\n' \
|
||||
"${service}" \
|
||||
"$(systemctl is-enabled "${service}" 2>/dev/null || true)" \
|
||||
"$(systemctl is-active "${service}" 2>/dev/null || true)"
|
||||
else
|
||||
printf '%s: not-installed\n' "${service}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
|
||||
echo "[5/8] runtime preflight"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/preflight"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[6/8] runtime readiness"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/readiness"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[7/8] runtime build-info"
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
|
||||
printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}"
|
||||
|
||||
echo "[8/8] condensed env audit summary"
|
||||
"${PYTHON_BIN}" - <<'PY' \
|
||||
"${BASE_URL}" \
|
||||
"${API_SERVICE}" \
|
||||
"${WORKER_SERVICE}" \
|
||||
"${SYNC_AGENT_SERVICE}" \
|
||||
"${NODE_AGENT_SERVICE}" \
|
||||
"${API_ENV_FILE}" \
|
||||
"${WORKER_ENV_FILE}" \
|
||||
"${NODE_AGENT_ENV_FILE}" \
|
||||
"${PROJECT_ROOT}"
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from typing import Dict, List
|
||||
|
||||
base_url, api_service, worker_service, sync_agent_service, node_agent_service, api_env, worker_env, node_agent_env, project_root = sys.argv[1:10]
|
||||
project_root_path = pathlib.Path(project_root)
|
||||
|
||||
def has_module(name: str) -> bool:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
|
||||
def inspect_python_runtime(path: str) -> Dict[str, object]:
|
||||
runtime_path = pathlib.Path(path)
|
||||
if not runtime_path.exists():
|
||||
return {}
|
||||
probe = r"""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
|
||||
mods = ["pytest", "fastapi", "uvicorn"]
|
||||
print(json.dumps({
|
||||
"python_executable": sys.executable,
|
||||
"python_version": sys.version.split()[0],
|
||||
"modules": {name: importlib.util.find_spec(name) is not None for name in mods},
|
||||
}, ensure_ascii=False))
|
||||
"""
|
||||
try:
|
||||
raw = subprocess.check_output([str(runtime_path), "-c", probe], text=True, stderr=subprocess.DEVNULL)
|
||||
except Exception:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except Exception:
|
||||
return {}
|
||||
payload["path"] = str(runtime_path)
|
||||
return payload
|
||||
|
||||
def systemd_state(service: str) -> dict:
|
||||
try:
|
||||
enabled = subprocess.check_output(
|
||||
["systemctl", "is-enabled", service],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
enabled = ""
|
||||
try:
|
||||
active = subprocess.check_output(
|
||||
["systemctl", "is-active", service],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
active = ""
|
||||
return {"enabled": enabled, "active": active}
|
||||
|
||||
def fetch_json(url: str) -> tuple[str, dict]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as resp:
|
||||
status = str(resp.status)
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return "000", {}
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except Exception:
|
||||
return status, {}
|
||||
return status, payload.get("data") if isinstance(payload, dict) else {}
|
||||
|
||||
def read_git_value(*args: str) -> str:
|
||||
try:
|
||||
result = subprocess.check_output(
|
||||
["git", "-C", str(project_root_path), *args],
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
).strip()
|
||||
except Exception:
|
||||
return ""
|
||||
return result
|
||||
|
||||
def read_text(path: pathlib.Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
local_commit_sha = read_git_value("rev-parse", "--short=12", "HEAD")
|
||||
local_commit_ref = read_git_value("rev-parse", "--abbrev-ref", "HEAD")
|
||||
ops_agent_service_text = read_text(project_root_path / "domain-api" / "app" / "services" / "ops_agent_service.py")
|
||||
bootstrap_node_agent_text = read_text(project_root_path / "domain-api" / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
|
||||
|
||||
repo_supports_install_command_block = (
|
||||
"install_command_block" in ops_agent_service_text
|
||||
and "_candidate_node_agent_install_paths" in ops_agent_service_text
|
||||
)
|
||||
repo_supports_multi_layout_bootstrap = "resolve_project_root" in bootstrap_node_agent_text
|
||||
|
||||
preflight_status, preflight = fetch_json(f"{base_url}/api/v1/runtime/preflight")
|
||||
readiness_status, readiness = fetch_json(f"{base_url}/api/v1/runtime/readiness")
|
||||
build_status, build = fetch_json(f"{base_url}/api/v1/runtime/build-info")
|
||||
route_surface = build.get("route_surface") if isinstance(build, dict) else {}
|
||||
|
||||
missing = []
|
||||
tooling_items: List[str] = []
|
||||
for env_file in [api_env, worker_env, node_agent_env]:
|
||||
if not pathlib.Path(env_file).exists():
|
||||
missing.append(f"missing_env:{env_file}")
|
||||
|
||||
python_candidates: List[str] = []
|
||||
seen_candidates = set()
|
||||
for candidate in [
|
||||
"/opt/domaincheck/domainCheck/.venv/bin/python",
|
||||
str(pathlib.Path.cwd() / ".venv" / "bin" / "python"),
|
||||
str(pathlib.Path.cwd() / "domain-api" / ".venv" / "bin" / "python"),
|
||||
pathlib.Path(sys.executable).as_posix(),
|
||||
]:
|
||||
if candidate and candidate not in seen_candidates:
|
||||
seen_candidates.add(candidate)
|
||||
python_candidates.append(candidate)
|
||||
|
||||
python_runtimes = [item for item in (inspect_python_runtime(candidate) for candidate in python_candidates) if item]
|
||||
|
||||
def runtime_has_module(module_name: str) -> bool:
|
||||
for runtime in python_runtimes:
|
||||
modules = runtime.get("modules") or {}
|
||||
if bool(modules.get(module_name)):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not runtime_has_module("fastapi"):
|
||||
missing.append("python_module:fastapi")
|
||||
if not runtime_has_module("uvicorn"):
|
||||
missing.append("python_module:uvicorn")
|
||||
if not runtime_has_module("pytest"):
|
||||
tooling_items.append("python_module:pytest")
|
||||
|
||||
runtime_commit_sha = str(build.get("commit_sha") or "").strip() if isinstance(build, dict) else ""
|
||||
runtime_commit_ref = str(build.get("commit_ref") or "").strip() if isinstance(build, dict) else ""
|
||||
runtime_route_missing_keys = [str(item).strip() for item in list((route_surface or {}).get("missing_keys") or []) if str(item).strip()]
|
||||
runtime_expected_paths = {
|
||||
str(key).strip(): str(value).strip()
|
||||
for key, value in dict((route_surface or {}).get("expected_paths") or {}).items()
|
||||
if str(key).strip()
|
||||
}
|
||||
runtime_bootstrap_plan_route_missing = "ops_node_handover_bootstrap_plan" in runtime_route_missing_keys
|
||||
runtime_declares_bootstrap_plan_route = "ops_node_handover_bootstrap_plan" in runtime_expected_paths
|
||||
runtime_repo_commit_drift = bool(local_commit_sha and runtime_commit_sha and local_commit_sha != runtime_commit_sha)
|
||||
runtime_repo_capability_drift = bool(
|
||||
repo_supports_install_command_block
|
||||
and (runtime_bootstrap_plan_route_missing or not runtime_declares_bootstrap_plan_route)
|
||||
)
|
||||
|
||||
recommended_actions: List[str] = []
|
||||
if runtime_repo_capability_drift:
|
||||
recommended_actions.append("systemctl restart domaincheck-api")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01")
|
||||
elif runtime_repo_commit_drift:
|
||||
recommended_actions.append("systemctl restart domaincheck-api")
|
||||
recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100")
|
||||
|
||||
payload = {
|
||||
"status": "ready",
|
||||
"headline": "当前环境没有发现新的基础阻断,可继续做上线复核。",
|
||||
"repo": {
|
||||
"project_root": str(project_root_path),
|
||||
"commit_sha": local_commit_sha,
|
||||
"commit_ref": local_commit_ref,
|
||||
"supports_install_command_block": repo_supports_install_command_block,
|
||||
"supports_multi_layout_bootstrap": repo_supports_multi_layout_bootstrap,
|
||||
},
|
||||
"python": {
|
||||
"pytest_installed": runtime_has_module("pytest"),
|
||||
"fastapi_installed": runtime_has_module("fastapi"),
|
||||
"uvicorn_installed": runtime_has_module("uvicorn"),
|
||||
"runtimes": python_runtimes,
|
||||
},
|
||||
"services": {
|
||||
api_service: systemd_state(api_service),
|
||||
worker_service: systemd_state(worker_service),
|
||||
sync_agent_service: systemd_state(sync_agent_service),
|
||||
node_agent_service: systemd_state(node_agent_service),
|
||||
},
|
||||
"runtime": {
|
||||
"preflight_status_code": preflight_status,
|
||||
"preflight_ok": bool((preflight or {}).get("ok", False)),
|
||||
"readiness_status_code": readiness_status,
|
||||
"readiness_status": str((readiness or {}).get("status") or ""),
|
||||
"readiness_summary": str((readiness or {}).get("summary") or ""),
|
||||
"build_status_code": build_status,
|
||||
"build_commit_sha": runtime_commit_sha,
|
||||
"build_commit_ref": runtime_commit_ref,
|
||||
"route_surface_complete": bool((route_surface or {}).get("surface_complete", False)),
|
||||
"route_surface_missing_keys": runtime_route_missing_keys,
|
||||
"route_surface_declares_bootstrap_plan": runtime_declares_bootstrap_plan_route,
|
||||
"repo_commit_drift": runtime_repo_commit_drift,
|
||||
"repo_capability_drift": runtime_repo_capability_drift,
|
||||
"runtime_may_need_restart": runtime_repo_commit_drift or runtime_repo_capability_drift,
|
||||
},
|
||||
"missing_items": missing,
|
||||
"tooling_items": tooling_items,
|
||||
"recommended_actions": recommended_actions,
|
||||
}
|
||||
|
||||
if not bool((preflight or {}).get("ok", False)):
|
||||
payload["status"] = "blocked"
|
||||
payload["headline"] = "runtime/preflight 未通过,先修环境与依赖,再继续上线复核。"
|
||||
elif str((readiness or {}).get("status") or "") == "blocking":
|
||||
payload["status"] = "blocked"
|
||||
payload["headline"] = "runtime/readiness 当前为 blocking,现场仍不适合进入正式发布。"
|
||||
elif runtime_repo_capability_drift:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "本地仓库已经具备新的 Node Agent 接管能力,但运行中的 API 路由面仍旧,当前更像是服务未重启到最新代码。"
|
||||
elif runtime_repo_commit_drift:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "本地仓库 commit 与运行中 API build-info 不一致,建议先重启或重发当前版本,再继续上线复核。"
|
||||
elif not bool((route_surface or {}).get("surface_complete", False)):
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "运行中 API 路由面还不完整,建议先确认当前实例是否已经吃到最新版本。"
|
||||
elif missing:
|
||||
payload["status"] = "attention"
|
||||
payload["headline"] = "基础环境存在缺口,虽然未必阻断运行,但建议先补齐后再交付。"
|
||||
elif tooling_items:
|
||||
payload["status"] = "ready"
|
||||
payload["headline"] = "运行环境可继续上线复核,但本机工具链仍有缺口,建议后续补齐。"
|
||||
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
423
domain-api/deploy/multi-region/check_go_live.sh
Normal file
423
domain-api/deploy/multi-region/check_go_live.sh
Normal file
@@ -0,0 +1,423 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
OVERSEAS_BASE_URL="${2:-}"
|
||||
VIEW_MODE="${3:-full}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-20}"
|
||||
LOCAL_NODE_AGENT_SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
|
||||
LOCAL_NODE_AGENT_ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
||||
|
||||
FETCH_BODY=""
|
||||
FETCH_STATUS="000"
|
||||
|
||||
fetch_json() {
|
||||
local url="${1:-}"
|
||||
local raw_text
|
||||
raw_text="$(curl -sS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"${url}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
||||
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
|
||||
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
|
||||
}
|
||||
|
||||
run_local_node_agent_probe() {
|
||||
SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME}" \
|
||||
ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE}" \
|
||||
bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check_node_agent.sh"
|
||||
}
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}"
|
||||
STACK_JSON="${FETCH_BODY}"
|
||||
STACK_STATUS="${FETCH_STATUS}"
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/go-live-summary?base_url=${BASE_URL}"
|
||||
GO_LIVE_API_JSON="${FETCH_BODY}"
|
||||
GO_LIVE_API_STATUS="${FETCH_STATUS}"
|
||||
|
||||
if [[ "${GO_LIVE_API_STATUS}" == 2* ]]; then
|
||||
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
||||
echo "[1/1] go-live summary (api)"
|
||||
fi
|
||||
printf '%s\n' "${GO_LIVE_API_JSON}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/contracts"
|
||||
CONTRACTS_JSON="${FETCH_BODY}"
|
||||
CONTRACTS_STATUS="${FETCH_STATUS}"
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad"
|
||||
LAUNCHPAD_JSON="${FETCH_BODY}"
|
||||
LAUNCHPAD_STATUS="${FETCH_STATUS}"
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/runtime/build-info"
|
||||
BUILD_INFO_JSON="${FETCH_BODY}"
|
||||
BUILD_INFO_STATUS="${FETCH_STATUS}"
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/nodes"
|
||||
NODES_JSON="${FETCH_BODY}"
|
||||
NODES_STATUS="${FETCH_STATUS}"
|
||||
|
||||
fetch_json "${BASE_URL}/api/v1/ops/overview"
|
||||
OVERVIEW_JSON="${FETCH_BODY}"
|
||||
OVERVIEW_STATUS="${FETCH_STATUS}"
|
||||
|
||||
LOCAL_NODE_AGENT_RAW="$(run_local_node_agent_probe 2>&1 || true)"
|
||||
|
||||
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
||||
echo "[1/7] ops stack diagnosis"
|
||||
printf 'status=%s\n%s\n\n' "${STACK_STATUS}" "${STACK_JSON}"
|
||||
|
||||
echo "[2/7] ops contracts"
|
||||
printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}"
|
||||
|
||||
echo "[3/7] release launchpad"
|
||||
printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}"
|
||||
|
||||
echo "[4/7] runtime build info"
|
||||
printf 'status=%s\n%s\n\n' "${BUILD_INFO_STATUS}" "${BUILD_INFO_JSON}"
|
||||
|
||||
echo "[5/7] managed nodes"
|
||||
printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}"
|
||||
|
||||
echo "[6/7] ops overview"
|
||||
printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}"
|
||||
|
||||
echo "[7/7] local node agent probe"
|
||||
printf '%s\n\n' "${LOCAL_NODE_AGENT_RAW}"
|
||||
fi
|
||||
|
||||
echo "[8/8] go-live summary"
|
||||
python3 - <<'PY' \
|
||||
"${BASE_URL}" \
|
||||
"${OVERSEAS_BASE_URL}" \
|
||||
"${STACK_JSON}" \
|
||||
"${STACK_STATUS}" \
|
||||
"${CONTRACTS_JSON}" \
|
||||
"${CONTRACTS_STATUS}" \
|
||||
"${LAUNCHPAD_JSON}" \
|
||||
"${LAUNCHPAD_STATUS}" \
|
||||
"${BUILD_INFO_JSON}" \
|
||||
"${BUILD_INFO_STATUS}" \
|
||||
"${NODES_JSON}" \
|
||||
"${NODES_STATUS}" \
|
||||
"${OVERVIEW_JSON}" \
|
||||
"${OVERVIEW_STATUS}" \
|
||||
"${LOCAL_NODE_AGENT_RAW}" \
|
||||
"${LOCAL_NODE_AGENT_SERVICE_NAME}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def load_payload(raw_text: str) -> dict:
|
||||
try:
|
||||
payload = json.loads(str(raw_text or ""))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_data(raw_text: str) -> dict:
|
||||
payload = load_payload(raw_text)
|
||||
data = payload.get("data")
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def ensure_dict(value):
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def ensure_list(value):
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def ensure_str(value):
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def truthy_http(status_code: str) -> bool:
|
||||
return str(status_code or "").startswith("2")
|
||||
|
||||
|
||||
base_url = str(sys.argv[1] or "http://127.0.0.1:8100").strip()
|
||||
overseas_base_url = str(sys.argv[2] or "").strip()
|
||||
stack_payload = load_data(sys.argv[3])
|
||||
stack_status_code = str(sys.argv[4] or "000")
|
||||
contracts_payload = load_data(sys.argv[5])
|
||||
contracts_status_code = str(sys.argv[6] or "000")
|
||||
launchpad_payload = load_data(sys.argv[7])
|
||||
launchpad_status_code = str(sys.argv[8] or "000")
|
||||
build_payload = load_data(sys.argv[9])
|
||||
build_status_code = str(sys.argv[10] or "000")
|
||||
nodes_payload = load_data(sys.argv[11])
|
||||
nodes_status_code = str(sys.argv[12] or "000")
|
||||
overview_payload = load_data(sys.argv[13])
|
||||
overview_status_code = str(sys.argv[14] or "000")
|
||||
local_node_agent_raw = str(sys.argv[15] or "")
|
||||
local_node_agent_service_name = str(sys.argv[16] or "domaincheck-node-agent").strip()
|
||||
|
||||
diagnosis = ensure_dict(stack_payload.get("diagnosis"))
|
||||
release_hub = ensure_dict(stack_payload.get("release_hub"))
|
||||
managed_nodes = ensure_dict(stack_payload.get("managed_nodes"))
|
||||
runtime_build_info = ensure_dict(stack_payload.get("runtime_build_info")) or build_payload
|
||||
launchpad_status = ensure_dict(launchpad_payload.get("launchpad_status"))
|
||||
launchpad_recommended_target_node_code = ensure_str(launchpad_status.get("recommended_target_node_code"))
|
||||
launchpad_recommended_recovery_label = ensure_str(launchpad_status.get("recommended_recovery_label"))
|
||||
launchpad_recommended_recovery_summary = ensure_str(launchpad_status.get("recommended_recovery_summary"))
|
||||
launchpad_onboarding_bootstrap_pending_nodes = int(
|
||||
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
|
||||
)
|
||||
launchpad_onboarding_acceptance_ready_nodes = int(
|
||||
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
|
||||
)
|
||||
contracts = ensure_list(contracts_payload.get("contracts"))
|
||||
nodes_summary = ensure_dict(nodes_payload.get("summary")) or ensure_dict(managed_nodes.get("summary"))
|
||||
route_surface = ensure_dict(runtime_build_info.get("route_surface"))
|
||||
next_step = ensure_dict(diagnosis.get("next_step"))
|
||||
operator_decision = ensure_dict(diagnosis.get("operator_decision"))
|
||||
overview_recommendation = ensure_dict(overview_payload.get("recommendation"))
|
||||
execution_scene = ensure_dict(overview_payload.get("execution_scene"))
|
||||
log_sync = ensure_dict(execution_scene.get("log_sync"))
|
||||
|
||||
local_agent_enabled = ""
|
||||
local_agent_active = ""
|
||||
local_agent_installed = False
|
||||
for line in [item.strip() for item in local_node_agent_raw.splitlines() if item.strip()]:
|
||||
if line == "not-installed":
|
||||
local_agent_installed = False
|
||||
if line == "enabled":
|
||||
local_agent_enabled = "enabled"
|
||||
local_agent_installed = True
|
||||
elif line == "disabled":
|
||||
local_agent_enabled = "disabled"
|
||||
local_agent_installed = True
|
||||
elif line in {"active", "inactive", "failed"}:
|
||||
local_agent_active = line
|
||||
local_agent_installed = True
|
||||
|
||||
blocking_reasons = []
|
||||
warnings = []
|
||||
publish_blocking_reasons = []
|
||||
publish_warnings = []
|
||||
|
||||
stack_status = ensure_str(diagnosis.get("stack_status")) or "unknown"
|
||||
if stack_status == "blocked":
|
||||
blocking_reasons.append("stack_diagnosis=blocked")
|
||||
publish_blocking_reasons.append("stack_diagnosis=blocked")
|
||||
elif stack_status == "attention":
|
||||
warnings.append("stack_diagnosis=attention")
|
||||
publish_warnings.append("stack_diagnosis=attention")
|
||||
elif not truthy_http(stack_status_code):
|
||||
blocking_reasons.append("stack_diagnosis_unavailable")
|
||||
publish_blocking_reasons.append("stack_diagnosis_unavailable")
|
||||
|
||||
surface_complete = bool(route_surface.get("surface_complete", False))
|
||||
if not surface_complete:
|
||||
missing_keys = [ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)]
|
||||
blocking_reasons.append(
|
||||
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
|
||||
)
|
||||
publish_blocking_reasons.append(
|
||||
"route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "")
|
||||
)
|
||||
|
||||
if not truthy_http(contracts_status_code) or len(contracts) <= 0:
|
||||
blocking_reasons.append("contracts_unavailable")
|
||||
publish_blocking_reasons.append("contracts_unavailable")
|
||||
|
||||
launchpad_status_code_value = ensure_str(launchpad_status.get("status")) or ensure_str(release_hub.get("launchpad_status")) or ""
|
||||
if launchpad_status_code_value == "blocked":
|
||||
blocking_reasons.append("release_launchpad=blocked")
|
||||
publish_blocking_reasons.append("release_launchpad=blocked")
|
||||
elif launchpad_status_code_value == "attention":
|
||||
warnings.append("release_launchpad=attention")
|
||||
publish_warnings.append("release_launchpad=attention")
|
||||
elif not truthy_http(launchpad_status_code):
|
||||
blocking_reasons.append("release_launchpad_unavailable")
|
||||
publish_blocking_reasons.append("release_launchpad_unavailable")
|
||||
|
||||
if launchpad_onboarding_bootstrap_pending_nodes > 0:
|
||||
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
||||
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
||||
if launchpad_onboarding_acceptance_ready_nodes > 0:
|
||||
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
||||
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
||||
|
||||
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
|
||||
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
|
||||
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
|
||||
if managed_enabled > 0 and remote_access_ready == 0:
|
||||
blocking_reasons.append("remote_agent_ready=0")
|
||||
publish_blocking_reasons.append("remote_agent_ready=0")
|
||||
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
|
||||
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
||||
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
||||
|
||||
if queue_dead_letter_nodes > 0:
|
||||
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
||||
publish_warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
||||
|
||||
if local_agent_installed and local_agent_active not in {"", "active"}:
|
||||
warnings.append(f"{local_node_agent_service_name}={local_agent_active}")
|
||||
|
||||
log_sync_enabled = bool(log_sync.get("enabled"))
|
||||
log_sync_state = ensure_str(log_sync.get("state"))
|
||||
log_sync_mode = ensure_str(log_sync.get("mode"))
|
||||
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
|
||||
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||||
missing_participating_node_codes = [
|
||||
ensure_str(item)
|
||||
for item in ensure_list(log_sync.get("missing_participating_nodes"))
|
||||
if ensure_str(item)
|
||||
]
|
||||
if log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes == 0:
|
||||
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
||||
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
||||
elif log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes < participating_nodes_total:
|
||||
warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
|
||||
publish_warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}")
|
||||
|
||||
operator_lane = ensure_str(operator_decision.get("lane"))
|
||||
operator_title = ensure_str(operator_decision.get("title"))
|
||||
operator_primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
|
||||
if not operator_lane:
|
||||
if blocking_reasons:
|
||||
operator_lane = "recovery"
|
||||
elif warnings:
|
||||
operator_lane = "verification"
|
||||
else:
|
||||
operator_lane = "go-live"
|
||||
if not operator_title:
|
||||
operator_title = (
|
||||
ensure_str(overview_recommendation.get("title"))
|
||||
or ensure_str(launchpad_status.get("status_label"))
|
||||
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
|
||||
)
|
||||
|
||||
next_step_action_code = (
|
||||
ensure_str(next_step.get("action_code"))
|
||||
or ensure_str(overview_recommendation.get("primary_action_code"))
|
||||
or ensure_str(launchpad_status.get("recommended_action_code"))
|
||||
)
|
||||
next_step_reason = (
|
||||
ensure_str(next_step.get("reason"))
|
||||
or ensure_str(overview_recommendation.get("reason"))
|
||||
or "来自 launchpad/overview fallback"
|
||||
)
|
||||
if not operator_primary_command_key:
|
||||
operator_primary_command_key = next_step_action_code
|
||||
|
||||
go_live_status = "ready"
|
||||
if blocking_reasons:
|
||||
go_live_status = "blocked"
|
||||
elif warnings:
|
||||
go_live_status = "attention"
|
||||
|
||||
publish_status = "ready"
|
||||
if publish_blocking_reasons:
|
||||
publish_status = "blocked"
|
||||
elif publish_warnings:
|
||||
publish_status = "attention"
|
||||
publish_ready = publish_status == "ready"
|
||||
if publish_status == "ready":
|
||||
publish_status_label = "可发布"
|
||||
elif publish_status == "attention":
|
||||
publish_status_label = "可发布但建议先复核"
|
||||
else:
|
||||
publish_status_label = "暂不可发布"
|
||||
|
||||
action_commands = {
|
||||
"handover_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} handover_first_gap",
|
||||
"view_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} view_first_gap",
|
||||
"bootstrap_run": (
|
||||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {launchpad_recommended_target_node_code} cli"
|
||||
if launchpad_recommended_target_node_code
|
||||
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} bootstrap_run"
|
||||
),
|
||||
"run_acceptance": (
|
||||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {launchpad_recommended_target_node_code} cli"
|
||||
if launchpad_recommended_target_node_code
|
||||
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_acceptance"
|
||||
),
|
||||
"fix_managed_nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} fix_managed_nodes",
|
||||
"open_worker_logs_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} open_worker_logs_participating",
|
||||
"run_inspection_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_inspection_participating",
|
||||
}
|
||||
|
||||
recommended_commands = {
|
||||
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary",
|
||||
"contracts": f"bash domain-api/deploy/multi-region/check_ops_contracts.sh {base_url}",
|
||||
"ops_plane": f"bash domain-api/deploy/multi-region/check_ops_plane.sh {base_url}",
|
||||
"release_hub": f"bash domain-api/deploy/multi-region/check_release_hub.sh {base_url}",
|
||||
"inspection": f"bash domain-api/deploy/multi-region/check_ops_inspection.sh {base_url}",
|
||||
"node_agent": "bash domain-api/deploy/multi-region/check_node_agent.sh",
|
||||
"overview": f"bash domain-api/deploy/multi-region/drive_ops_center.sh overview {base_url}",
|
||||
"doctor_export": (
|
||||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url} {overseas_base_url}"
|
||||
if overseas_base_url
|
||||
else f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url}"
|
||||
),
|
||||
}
|
||||
if next_step_action_code and next_step_action_code in action_commands:
|
||||
recommended_commands["next_step"] = action_commands[next_step_action_code]
|
||||
if log_sync_enabled:
|
||||
if "open_worker_logs_participating" in action_commands:
|
||||
recommended_commands["log_sync_logs"] = action_commands["open_worker_logs_participating"]
|
||||
if "run_inspection_participating" in action_commands:
|
||||
recommended_commands["log_sync_inspection"] = action_commands["run_inspection_participating"]
|
||||
|
||||
summary = {
|
||||
"base_url": base_url,
|
||||
"overseas_base_url": overseas_base_url,
|
||||
"go_live_status": go_live_status,
|
||||
"publish_ready": publish_ready,
|
||||
"publish_status": publish_status,
|
||||
"publish_status_label": publish_status_label,
|
||||
"stack_status": stack_status,
|
||||
"contracts_ready": truthy_http(contracts_status_code) and len(contracts) > 0,
|
||||
"contracts_total": len(contracts),
|
||||
"launchpad_status": launchpad_status_code_value,
|
||||
"launchpad_status_label": ensure_str(launchpad_status.get("status_label")),
|
||||
"launchpad_recommended_action_code": ensure_str(launchpad_status.get("recommended_action_code")),
|
||||
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
|
||||
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
|
||||
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
|
||||
"route_surface_complete": surface_complete,
|
||||
"route_surface_missing_keys": [
|
||||
ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)
|
||||
],
|
||||
"managed_enabled": managed_enabled,
|
||||
"remote_access_ready": remote_access_ready,
|
||||
"queue_dead_letter_nodes": queue_dead_letter_nodes,
|
||||
"participating_nodes_total": participating_nodes_total,
|
||||
"log_sync_enabled": log_sync_enabled,
|
||||
"log_sync_state": log_sync_state,
|
||||
"log_sync_mode": log_sync_mode,
|
||||
"log_sync_covered_nodes": covered_participating_nodes,
|
||||
"log_sync_missing_node_codes": missing_participating_node_codes,
|
||||
"next_step_action_code": next_step_action_code,
|
||||
"next_step_reason": next_step_reason,
|
||||
"operator_lane": operator_lane,
|
||||
"operator_title": operator_title,
|
||||
"operator_primary_command_key": operator_primary_command_key,
|
||||
"local_node_agent": {
|
||||
"service_name": local_node_agent_service_name,
|
||||
"installed": local_agent_installed,
|
||||
"enabled": local_agent_enabled,
|
||||
"active": local_agent_active,
|
||||
},
|
||||
"blocking_reasons": blocking_reasons,
|
||||
"warnings": warnings,
|
||||
"publish_blocking_reasons": publish_blocking_reasons,
|
||||
"publish_warnings": publish_warnings,
|
||||
"recommended_commands": recommended_commands,
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
182
domain-api/deploy/multi-region/check_go_live_signoff.sh
Normal file
182
domain-api/deploy/multi-region/check_go_live_signoff.sh
Normal file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib_ops_center.sh"
|
||||
ops_center_load_config "${SCRIPT_DIR}"
|
||||
|
||||
TARGET_OR_BASE_URL="${1:-}"
|
||||
SECOND_ARG="${2:-}"
|
||||
|
||||
REPORT_DIR=""
|
||||
MANIFEST_PATH=""
|
||||
GENERATED_BUNDLE="false"
|
||||
MAINLAND_BASE_URL=""
|
||||
OVERSEAS_BASE_URL=""
|
||||
|
||||
if [[ -n "${TARGET_OR_BASE_URL}" && ( -f "${TARGET_OR_BASE_URL}" || -d "${TARGET_OR_BASE_URL}" ) ]]; then
|
||||
if [[ -d "${TARGET_OR_BASE_URL}" ]]; then
|
||||
REPORT_DIR="${TARGET_OR_BASE_URL%/}"
|
||||
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
|
||||
else
|
||||
MANIFEST_PATH="${TARGET_OR_BASE_URL}"
|
||||
REPORT_DIR="$(cd "$(dirname "${MANIFEST_PATH}")" && pwd)"
|
||||
fi
|
||||
else
|
||||
MAINLAND_BASE_URL="${TARGET_OR_BASE_URL:-$(ops_center_resolve_mainland_api_base_url)}"
|
||||
OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${SECOND_ARG:-}" "$(ops_center_resolve_overseas_api_base_url)")"
|
||||
REPORT_DIR="${OPS_CENTER_REPORT_ROOT}/go-live-bundles/go-live-signoff-$(timestamp_now)"
|
||||
ops_center_ensure_report_dir "${REPORT_DIR}"
|
||||
bash "${SCRIPT_DIR}/export_go_live_bundle.sh" "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" >/dev/null
|
||||
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
|
||||
GENERATED_BUNDLE="true"
|
||||
fi
|
||||
|
||||
if [[ ! -f "${MANIFEST_PATH}" ]]; then
|
||||
echo "manifest not found: ${MANIFEST_PATH}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REVIEW_JSON="$(bash "${SCRIPT_DIR}/review_go_live_bundle.sh" "${MANIFEST_PATH}")"
|
||||
DOCTOR_JSON="$(bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${MANIFEST_PATH}")"
|
||||
|
||||
python3 - <<'PY' \
|
||||
"${REVIEW_JSON}" \
|
||||
"${DOCTOR_JSON}" \
|
||||
"${GENERATED_BUNDLE}" \
|
||||
"${REPORT_DIR}" \
|
||||
"${MANIFEST_PATH}" \
|
||||
"${MAINLAND_BASE_URL}" \
|
||||
"${OVERSEAS_BASE_URL}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
review = json.loads(sys.argv[1])
|
||||
doctor = json.loads(sys.argv[2])
|
||||
generated_bundle = str(sys.argv[3]).strip().lower() == "true"
|
||||
report_dir = str(sys.argv[4] or "").strip()
|
||||
manifest_path = str(sys.argv[5] or "").strip()
|
||||
mainland_base_url = str(sys.argv[6] or "").strip()
|
||||
overseas_base_url = str(sys.argv[7] or "").strip()
|
||||
|
||||
go_live_summary = dict(review.get("go_live_summary") or {})
|
||||
stack_diagnosis = dict(review.get("stack_diagnosis") or {})
|
||||
driver_feed = dict(review.get("driver_feed") or {})
|
||||
codex_brief = dict(review.get("codex_brief") or {})
|
||||
launchpad_alignment = dict(review.get("launchpad_alignment") or {})
|
||||
doctor_decision = dict(doctor.get("decision") or {})
|
||||
doctor_summary = dict(doctor.get("summary") or {})
|
||||
|
||||
review_status = str(review.get("status") or "").strip()
|
||||
doctor_status = str(doctor_decision.get("status") or "").strip()
|
||||
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
|
||||
publish_status = str(go_live_summary.get("publish_status") or "").strip()
|
||||
stack_status = str(stack_diagnosis.get("stack_status") or "").strip()
|
||||
driver_launch_status = str(driver_feed.get("launch_status") or "").strip()
|
||||
codex_launch_status = str(codex_brief.get("launch_status") or "").strip()
|
||||
|
||||
|
||||
def dedupe_commands(values):
|
||||
result = []
|
||||
seen = set()
|
||||
for item in values:
|
||||
text = str(item or "").strip()
|
||||
if not text or text in seen:
|
||||
continue
|
||||
seen.add(text)
|
||||
result.append(text)
|
||||
return result
|
||||
|
||||
|
||||
blocked_reasons = dedupe_commands(
|
||||
[]
|
||||
+ ([f"review:{review_status}"] if review_status == "blocked" else [])
|
||||
+ ([f"doctor:{doctor_status}"] if doctor_status == "blocked" else [])
|
||||
+ ([f"go_live:{go_live_status}"] if go_live_status == "blocked" else [])
|
||||
+ ([f"publish:{publish_status}"] if publish_status == "blocked" else [])
|
||||
+ ([f"stack:{stack_status}"] if stack_status == "blocked" else [])
|
||||
+ ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "blocked" else [])
|
||||
+ ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "blocked" else [])
|
||||
)
|
||||
|
||||
attention_reasons = dedupe_commands(
|
||||
[]
|
||||
+ ([f"review:{review_status}"] if review_status == "attention" else [])
|
||||
+ ([f"doctor:{doctor_status}"] if doctor_status == "attention" else [])
|
||||
+ ([f"go_live:{go_live_status}"] if go_live_status == "attention" else [])
|
||||
+ ([f"publish:{publish_status}"] if publish_status == "attention" else [])
|
||||
+ ([f"stack:{stack_status}"] if stack_status == "attention" else [])
|
||||
+ ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "attention" else [])
|
||||
+ ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "attention" else [])
|
||||
+ (["launchpad_alignment:inconsistent"] if not bool(launchpad_alignment.get("consistent", False)) else [])
|
||||
)
|
||||
|
||||
if blocked_reasons:
|
||||
signoff_status = "blocked"
|
||||
headline = (
|
||||
str(review.get("headline") or "").strip()
|
||||
or str(doctor_decision.get("headline") or "").strip()
|
||||
or "当前发布前验证仍存在阻断项,不应直接签字上线。"
|
||||
)
|
||||
elif attention_reasons:
|
||||
signoff_status = "attention"
|
||||
headline = (
|
||||
str(doctor_decision.get("headline") or "").strip()
|
||||
or str(review.get("headline") or "").strip()
|
||||
or "当前现场已进入收口区间,但仍建议先完成 attention 项再正式签字。"
|
||||
)
|
||||
else:
|
||||
signoff_status = "ready"
|
||||
headline = "当前证据包、doctor 结论与发布门禁一致,可进入最终人工签字或正式发布。"
|
||||
|
||||
recommended_commands = dedupe_commands(
|
||||
[]
|
||||
+ list(doctor_decision.get("recommended_commands") or [])
|
||||
+ [
|
||||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review {manifest_path}",
|
||||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision {manifest_path}",
|
||||
]
|
||||
)
|
||||
|
||||
payload = {
|
||||
"signoff_status": signoff_status,
|
||||
"headline": headline,
|
||||
"generated_bundle": generated_bundle,
|
||||
"report_dir": report_dir or str(doctor.get("report_dir") or review.get("report_dir") or ""),
|
||||
"manifest_path": manifest_path or str(doctor.get("manifest_path") or ""),
|
||||
"base_urls": {
|
||||
"mainland": mainland_base_url,
|
||||
"overseas": overseas_base_url,
|
||||
},
|
||||
"release_gate": {
|
||||
"go_live_status": go_live_status,
|
||||
"publish_status": publish_status,
|
||||
"stack_status": stack_status,
|
||||
"driver_launch_status": driver_launch_status,
|
||||
"codex_launch_status": codex_launch_status,
|
||||
"bundle_review_status": review_status,
|
||||
"doctor_status": doctor_status,
|
||||
},
|
||||
"blocked_reasons": blocked_reasons,
|
||||
"attention_reasons": attention_reasons,
|
||||
"decision": {
|
||||
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
|
||||
"next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(),
|
||||
"doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(),
|
||||
"doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(),
|
||||
"launchpad_recommended_target_node_code": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
"launchpad_recommended_recovery_label": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
"launchpad_recommended_recovery_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
},
|
||||
"launchpad_alignment": launchpad_alignment,
|
||||
"recommended_commands": recommended_commands,
|
||||
"bundle_review": review,
|
||||
"doctor_decision": {
|
||||
"summary": doctor_summary,
|
||||
"decision": doctor_decision,
|
||||
},
|
||||
}
|
||||
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
38
domain-api/deploy/multi-region/check_node_agent.sh
Executable file
38
domain-api/deploy/multi-region/check_node_agent.sh
Executable file
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SERVICE_NAME="${SERVICE_NAME:-domaincheck-node-agent}"
|
||||
ENV_FILE="${ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
||||
|
||||
echo "[1/4] environment file"
|
||||
if [[ -f "${ENV_FILE}" ]]; then
|
||||
echo "env file: ${ENV_FILE}"
|
||||
else
|
||||
echo "missing env file: ${ENV_FILE}"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[2/4] key env summary"
|
||||
if [[ -f "${ENV_FILE}" ]]; then
|
||||
grep -E '^(OPS_CONTROL_PLANE_BASE_URL|NODE_CODE|NODE_REGION|NODE_ROLE|OPS_AGENT_POLL_INTERVAL_SECONDS|WORKER_SERVICE_NAME|API_SERVICE_NAME|SYNC_AGENT_SERVICE_NAME)=' "${ENV_FILE}" || true
|
||||
else
|
||||
echo "skip"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[3/4] systemd status"
|
||||
if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then
|
||||
systemctl is-enabled "${SERVICE_NAME}" || true
|
||||
systemctl is-active "${SERVICE_NAME}" || true
|
||||
else
|
||||
echo "not-installed"
|
||||
echo "inactive"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[4/4] service detail"
|
||||
if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then
|
||||
systemctl status "${SERVICE_NAME}" --no-pager -l | sed -n '1,25p' || true
|
||||
else
|
||||
echo "service not found: ${SERVICE_NAME}"
|
||||
fi
|
||||
65
domain-api/deploy/multi-region/check_node_scene_log.sh
Normal file
65
domain-api/deploy/multi-region/check_node_scene_log.sh
Normal file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
NODE_CODE="${2:-}"
|
||||
LIMIT="${3:-80}"
|
||||
MODE="${4:-key}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-8}"
|
||||
|
||||
if [[ -z "${NODE_CODE}" ]]; then
|
||||
echo "usage: bash domain-api/deploy/multi-region/check_node_scene_log.sh [base_url] <node_code> [limit] [key|full]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RAW_RESPONSE="$(curl -sS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"${BASE_URL}/api/v1/ops/nodes/${NODE_CODE}/scene-log?limit=${LIMIT}&mode=${MODE}")"
|
||||
|
||||
echo "[1/2] raw response"
|
||||
printf '%s\n\n' "${RAW_RESPONSE}"
|
||||
|
||||
echo "[2/2] condensed scene-log summary"
|
||||
python3 - "${RAW_RESPONSE}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
raw = str(sys.argv[1] or "")
|
||||
payload = json.loads(raw)
|
||||
data = dict(payload.get("data") or {})
|
||||
participation = dict(data.get("participation") or {})
|
||||
source_summary = dict(data.get("source_summary") or {})
|
||||
records = list(data.get("records") or [])
|
||||
latest = dict(data.get("latest_record") or {})
|
||||
|
||||
summary = {
|
||||
"node_code": str(data.get("node_code") or ""),
|
||||
"status": str(data.get("status") or ""),
|
||||
"status_label": str(data.get("status_label") or ""),
|
||||
"mode": str(data.get("mode") or ""),
|
||||
"summary": str(data.get("summary") or ""),
|
||||
"log_sync_enabled": bool(data.get("log_sync_enabled", False)),
|
||||
"records_total": int(data.get("records_total", len(records)) or 0),
|
||||
"missing_reason_code": str(data.get("missing_reason_code") or ""),
|
||||
"participation": {
|
||||
"detect_participating": bool(participation.get("detect_participating", False)),
|
||||
"participation_state": str(participation.get("participation_state") or ""),
|
||||
"participation_label": str(participation.get("participation_label") or ""),
|
||||
"participation_reason": str(participation.get("participation_reason") or ""),
|
||||
"current_load": int(participation.get("current_load", 0) or 0),
|
||||
},
|
||||
"source_summary": {
|
||||
"line_count": int(source_summary.get("line_count", 0) or 0),
|
||||
"key_line_count": int(source_summary.get("key_line_count", 0) or 0),
|
||||
"full_line_count": int(source_summary.get("full_line_count", 0) or 0),
|
||||
"last_at": str(source_summary.get("last_at") or ""),
|
||||
"last_line": str(source_summary.get("last_line") or ""),
|
||||
},
|
||||
"latest_record": latest,
|
||||
"sample_messages": [str(item.get("message") or "") for item in records[:8]],
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
1272
domain-api/deploy/multi-region/check_ops_center_stack.sh
Normal file
1272
domain-api/deploy/multi-region/check_ops_center_stack.sh
Normal file
File diff suppressed because it is too large
Load Diff
55
domain-api/deploy/multi-region/check_ops_contracts.sh
Executable file
55
domain-api/deploy/multi-region/check_ops_contracts.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
|
||||
contracts_json="$(curl -fsS "${BASE_URL}/api/v1/ops/contracts")"
|
||||
|
||||
echo "[1/3] ops contracts registry"
|
||||
printf '%s\n\n' "${contracts_json}"
|
||||
|
||||
echo "[2/3] condensed summary"
|
||||
python3 - <<'PY' "${contracts_json}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
data = payload.get("data") or {}
|
||||
contracts = list(data.get("contracts") or [])
|
||||
|
||||
summary = {
|
||||
"registry_version": str(data.get("registry_version") or ""),
|
||||
"schema_version": str(data.get("schema_version") or ""),
|
||||
"contracts_total": len(contracts),
|
||||
"contract_keys": [str(item.get("key") or "") for item in contracts if str(item.get("key") or "")],
|
||||
"docs_root": str(data.get("docs_root") or ""),
|
||||
"discovery_entrypoints": list(data.get("discovery_entrypoints") or []),
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[3/3] contract rows"
|
||||
python3 - <<'PY' "${contracts_json}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
data = payload.get("data") or {}
|
||||
contracts = list(data.get("contracts") or [])
|
||||
|
||||
rows = [
|
||||
{
|
||||
"key": str(item.get("key") or ""),
|
||||
"version": str(item.get("version") or ""),
|
||||
"status": str(item.get("status") or ""),
|
||||
"primary_endpoint": str(item.get("primary_endpoint") or ""),
|
||||
"schema_doc_path": str(item.get("schema_doc_path") or ""),
|
||||
"consumers": list(item.get("consumers") or []),
|
||||
}
|
||||
for item in contracts
|
||||
]
|
||||
|
||||
print(json.dumps(rows, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
72
domain-api/deploy/multi-region/check_ops_inspection.sh
Executable file
72
domain-api/deploy/multi-region/check_ops_inspection.sh
Executable file
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
|
||||
|
||||
curl_json() {
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"$1"
|
||||
}
|
||||
|
||||
INSPECTION_JSON="$(curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=120")"
|
||||
|
||||
echo "[1/5] inspection overview"
|
||||
printf '%s\n\n' "${INSPECTION_JSON}"
|
||||
|
||||
echo "[2/5] condensed summary"
|
||||
python3 - <<'PY' "${INSPECTION_JSON}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1]).get("data") or {}
|
||||
print(json.dumps({
|
||||
"rows_total": int(payload.get("rows_total") or 0),
|
||||
"filtered_rows_total": int(payload.get("filtered_rows_total") or 0),
|
||||
"visible_nodes_total": int(payload.get("visible_nodes_total") or 0),
|
||||
"eligible_nodes_total": int(payload.get("eligible_nodes_total") or 0),
|
||||
"handover_gap_nodes_total": int(payload.get("handover_gap_nodes_total") or 0),
|
||||
"participating_nodes_total": int(payload.get("participating_nodes_total") or 0),
|
||||
"summary": dict(payload.get("summary") or {}),
|
||||
"available_problem_kind_counts": dict(payload.get("available_problem_kind_counts") or {}),
|
||||
"filters": dict(payload.get("filters") or {}),
|
||||
}, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[3/5] priority queue"
|
||||
python3 - <<'PY' "${INSPECTION_JSON}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1]).get("data") or {}
|
||||
rows = list(payload.get("priority_queue") or [])
|
||||
print(json.dumps({
|
||||
"total": len(rows),
|
||||
"items": [
|
||||
{
|
||||
"node_code": str(item.get("node_code") or ""),
|
||||
"overall_status": str(item.get("overall_status") or ""),
|
||||
"problem_kind": str(item.get("problem_kind") or ""),
|
||||
"problem_label": str(item.get("problem_label") or ""),
|
||||
"recommended_action_code": str(item.get("recommended_action_code") or ""),
|
||||
"last_updated_at": str(item.get("last_updated_at") or ""),
|
||||
"issue_summary": str(item.get("issue_summary") or ""),
|
||||
}
|
||||
for item in rows[:12]
|
||||
],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[4/5] attention only"
|
||||
curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&status=attention&only_problem=true"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[5/5] participating problems only"
|
||||
curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&only_problem=true&only_participating=true"
|
||||
echo
|
||||
35
domain-api/deploy/multi-region/check_ops_link_snapshot.sh
Normal file
35
domain-api/deploy/multi-region/check_ops_link_snapshot.sh
Normal file
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
|
||||
echo "[1/2] ops link snapshot"
|
||||
SNAPSHOT_JSON="$(curl -fsS "${BASE_URL}/api/v1/ops/link-snapshot")"
|
||||
echo "${SNAPSHOT_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[2/2] condensed summary"
|
||||
"${PYTHON_BIN}" - <<'PY' "$SNAPSHOT_JSON"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1]).get("data", {})
|
||||
print(f"status: {payload.get('status', '-')}")
|
||||
print(f"headline: {payload.get('headline', '-')}")
|
||||
print()
|
||||
print("summary:")
|
||||
for line in payload.get("summary_lines", []) or []:
|
||||
print(f"- {line}")
|
||||
|
||||
actions = payload.get("next_actions", []) or []
|
||||
if actions:
|
||||
print()
|
||||
print("next_actions:")
|
||||
for item in actions:
|
||||
label = str(item.get("label", "") or "").strip() or "-"
|
||||
action_code = str(item.get("action_code", "") or "").strip() or "-"
|
||||
node_codes = "、".join(str(x).strip() for x in (item.get("node_codes", []) or []) if str(x).strip()) or "-"
|
||||
print(f"- {label} / {action_code} / nodes: {node_codes}")
|
||||
PY
|
||||
609
domain-api/deploy/multi-region/check_ops_plane.sh
Normal file
609
domain-api/deploy/multi-region/check_ops_plane.sh
Normal file
@@ -0,0 +1,609 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
|
||||
|
||||
curl_json() {
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"$1"
|
||||
}
|
||||
|
||||
curl_post_json() {
|
||||
local url="$1"
|
||||
local body="$2"
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
-X POST "${url}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "${body}"
|
||||
}
|
||||
|
||||
OVERVIEW_JSON="$(curl_json "${BASE_URL}/api/v1/ops/overview")"
|
||||
NODES_JSON="$(curl_json "${BASE_URL}/api/v1/ops/nodes")"
|
||||
JOBS_JSON="$(curl_json "${BASE_URL}/api/v1/ops/jobs")"
|
||||
ACTIVITY_JSON="$(curl_json "${BASE_URL}/api/v1/ops/activity-stream")"
|
||||
LAUNCHPAD_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")"
|
||||
LATEST_RELEASE_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")"
|
||||
POLICY_PREVIEW_JSON="$(
|
||||
curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" \
|
||||
'{"action":"runtime.restart_api","target_type":"node","target_node_code":"mainland-controller-01"}'
|
||||
)"
|
||||
|
||||
echo "[1/12] ops overview"
|
||||
printf '%s\n\n' "${OVERVIEW_JSON}"
|
||||
|
||||
echo "[2/12] top recommendation"
|
||||
python3 - <<'PY' "${OVERVIEW_JSON}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
data = payload.get("data") or {}
|
||||
recommendation = dict(data.get("recommendation") or {})
|
||||
print(json.dumps({
|
||||
"source": str(recommendation.get("source") or ""),
|
||||
"key": str(recommendation.get("key") or ""),
|
||||
"priority": str(recommendation.get("priority") or ""),
|
||||
"summary": str(recommendation.get("summary") or ""),
|
||||
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
|
||||
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
|
||||
"node_codes": list(recommendation.get("node_codes") or []),
|
||||
}, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[3/12] driver recommendations (condensed)"
|
||||
python3 - <<'PY' "${OVERVIEW_JSON}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
data = payload.get("data") or {}
|
||||
recommendations = list(data.get("driver_recommendations") or [])
|
||||
summary = {
|
||||
"total": len(recommendations),
|
||||
"items": [
|
||||
{
|
||||
"key": str(item.get("key") or ""),
|
||||
"title": str(item.get("title") or ""),
|
||||
"level_label": str(item.get("level_label") or ""),
|
||||
"primary_action_code": str(item.get("primary_action_code") or ""),
|
||||
"secondary_action_code": str(item.get("secondary_action_code") or ""),
|
||||
"meta_text": str(item.get("meta_text") or ""),
|
||||
}
|
||||
for item in recommendations[:8]
|
||||
],
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[4/12] ops capabilities"
|
||||
curl_json "${BASE_URL}/api/v1/ops/capabilities"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[5/12] ops blueprint"
|
||||
curl_json "${BASE_URL}/api/v1/ops/blueprint"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[6/12] ops runbook"
|
||||
curl_json "${BASE_URL}/api/v1/ops/runbook"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[7/12] ops nodes"
|
||||
printf '%s\n' "${NODES_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[8/12] ops jobs"
|
||||
printf '%s\n' "${JOBS_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[9/12] ops activity stream"
|
||||
printf '%s\n' "${ACTIVITY_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[10/12] release launchpad"
|
||||
printf '%s\n' "${LAUNCHPAD_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[11/12] latest release"
|
||||
printf '%s\n' "${LATEST_RELEASE_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[12/12] policy preview sample"
|
||||
printf '%s\n' "${POLICY_PREVIEW_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[13/13] condensed summary"
|
||||
python3 - <<'PY' \
|
||||
"${BASE_URL}" \
|
||||
"${OVERVIEW_JSON}" \
|
||||
"${NODES_JSON}" \
|
||||
"${JOBS_JSON}" \
|
||||
"${ACTIVITY_JSON}" \
|
||||
"${LAUNCHPAD_JSON}" \
|
||||
"${LATEST_RELEASE_JSON}" \
|
||||
"${POLICY_PREVIEW_JSON}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def as_dict(payload):
|
||||
try:
|
||||
raw = json.loads(payload)
|
||||
except Exception:
|
||||
return {}
|
||||
return raw.get("data") or {}
|
||||
|
||||
|
||||
def ensure_dict(value):
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def ensure_list(value):
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def ensure_str(value):
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def command(name, *parts):
|
||||
tokens = [
|
||||
"bash",
|
||||
"domain-api/deploy/multi-region/drive_ops_center.sh",
|
||||
name,
|
||||
base_url,
|
||||
]
|
||||
tokens.extend([str(part) for part in parts if str(part or "").strip()])
|
||||
return " ".join(tokens)
|
||||
|
||||
|
||||
def release_command(name, *parts):
|
||||
tokens = [
|
||||
"bash",
|
||||
"domain-api/deploy/multi-region/drive_release_hub.sh",
|
||||
base_url,
|
||||
name,
|
||||
]
|
||||
tokens.extend([str(part) for part in parts if str(part or "").strip()])
|
||||
return " ".join(tokens)
|
||||
|
||||
|
||||
def build_operator_decision():
|
||||
remote_access_state_counts = ensure_dict(node_summary.get("remote_access_state_counts"))
|
||||
pending_bootstrap = int(node_summary.get("pending_bootstrap", 0) or 0)
|
||||
runtime_only = int(node_summary.get("runtime_only", 0) or 0)
|
||||
online_total = int(node_summary.get("online", 0) or 0)
|
||||
participating_total = int(scene_log.get("participating_node_count", 0) or 0)
|
||||
launchpad_blocked = bool(launchpad_workers.get("blocked")) or bool(launchpad_controls.get("blocked"))
|
||||
running_total = int(jobs_summary.get("running_count", 0) or 0)
|
||||
queued_total = int(jobs_summary.get("queued_count", 0) or 0)
|
||||
|
||||
if scene_log.get("status") == "disabled" and participating_total > 0:
|
||||
return {
|
||||
"lane": "observability",
|
||||
"priority": "high",
|
||||
"reason_code": "scene_log_disabled",
|
||||
"title": "先恢复远端现场日志可见性",
|
||||
"summary": ensure_str(scene_log.get("summary")),
|
||||
"next_focus": ensure_dict(scene_log.get("focus_ref")),
|
||||
"primary_command_key": "scene_log",
|
||||
"secondary_command_key": "driver_focus_preview",
|
||||
}
|
||||
|
||||
if pending_bootstrap > 0 or remote_access_state_counts:
|
||||
return {
|
||||
"lane": "node_handover",
|
||||
"priority": "high",
|
||||
"reason_code": "managed_nodes_not_ready",
|
||||
"title": "先补齐节点接管与远程可达性",
|
||||
"summary": (
|
||||
f"当前 managed node 共 {int(node_summary.get('managed_total', 0) or 0)} 台,"
|
||||
f"pending_bootstrap={pending_bootstrap},runtime_only={runtime_only}。"
|
||||
),
|
||||
"next_focus": {
|
||||
"kind": "managed_node_handover",
|
||||
"node_code": first_attention_node_code,
|
||||
} if first_attention_node_code else {},
|
||||
"primary_command_key": "node_handover",
|
||||
"secondary_command_key": "node_bootstrap_plan",
|
||||
}
|
||||
|
||||
if launchpad_blocked or str(launchpad_status.get("status") or "") == "attention":
|
||||
return {
|
||||
"lane": "release",
|
||||
"priority": "medium",
|
||||
"reason_code": "release_launchpad_attention",
|
||||
"title": "检查发布驾驶舱与 rollout 阻断",
|
||||
"summary": ensure_str(launchpad_status.get("summary")),
|
||||
"next_focus": release_focus_ref,
|
||||
"primary_command_key": "release_launchpad",
|
||||
"secondary_command_key": "release_preview_latest",
|
||||
}
|
||||
|
||||
if running_total > 0 or queued_total > 0:
|
||||
return {
|
||||
"lane": "ops_jobs",
|
||||
"priority": "medium",
|
||||
"reason_code": "jobs_active",
|
||||
"title": "继续盯运行中任务与活动流",
|
||||
"summary": (
|
||||
f"当前 running={running_total},queued={queued_total},"
|
||||
f"activity_status={ensure_str(activity_summary.get('status')) or 'unknown'}。"
|
||||
),
|
||||
"next_focus": {
|
||||
"kind": "ops_jobs",
|
||||
"section": "activity_stream",
|
||||
},
|
||||
"primary_command_key": "jobs",
|
||||
"secondary_command_key": "activity_preview",
|
||||
}
|
||||
|
||||
return {
|
||||
"lane": "steady",
|
||||
"priority": "low",
|
||||
"reason_code": "steady_state",
|
||||
"title": "当前进入稳定观察态",
|
||||
"summary": (
|
||||
f"online={online_total},attention_nodes={len(attention_nodes)},"
|
||||
f"scene_log_status={ensure_str(scene_log.get('status')) or 'standby'}。"
|
||||
),
|
||||
"next_focus": recommendation_focus_ref or ensure_dict(scene_log.get("focus_ref")),
|
||||
"primary_command_key": "driver_feed",
|
||||
"secondary_command_key": "activity_preview",
|
||||
}
|
||||
|
||||
|
||||
def normalize_scene_log_observation(overview_data):
|
||||
driver_feed = ensure_dict(overview_data.get("driver_feed"))
|
||||
scene_log_data = ensure_dict(driver_feed.get("scene_log_observation"))
|
||||
if scene_log_data:
|
||||
normalized = dict(scene_log_data)
|
||||
normalized["status"] = ensure_str(normalized.get("status")) or "standby"
|
||||
normalized["status_label"] = ensure_str(normalized.get("status_label")) or "待命"
|
||||
normalized["summary"] = ensure_str(normalized.get("summary")) or "当前现场日志观察暂无额外结论。"
|
||||
normalized["focus_ref"] = ensure_dict(normalized.get("focus_ref"))
|
||||
normalized["target_node_code"] = ensure_str(normalized.get("target_node_code"))
|
||||
normalized["participating_node_count"] = int(normalized.get("participating_node_count", 0) or 0)
|
||||
normalized["covered_participating_node_count"] = int(normalized.get("covered_participating_node_count", 0) or 0)
|
||||
normalized["line_count"] = int(normalized.get("line_count", 0) or 0)
|
||||
return normalized
|
||||
|
||||
execution_scene = ensure_dict(overview_data.get("execution_scene"))
|
||||
log_sync = ensure_dict(execution_scene.get("log_sync"))
|
||||
participation = ensure_dict(execution_scene.get("participation_summary"))
|
||||
missing_nodes = ensure_list(log_sync.get("missing_participating_nodes"))
|
||||
participating_nodes = ensure_list(execution_scene.get("participating_nodes"))
|
||||
target_node_code = ""
|
||||
if missing_nodes:
|
||||
target_node_code = ensure_str(missing_nodes[0])
|
||||
elif participating_nodes:
|
||||
target_node_code = ensure_str(ensure_dict(participating_nodes[0]).get("node_code"))
|
||||
|
||||
participating_total = int(
|
||||
log_sync.get("participating_node_count", participation.get("participating_nodes", len(participating_nodes)))
|
||||
or participation.get("participating_nodes", len(participating_nodes))
|
||||
or len(participating_nodes)
|
||||
or 0
|
||||
)
|
||||
covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||||
line_count = int(log_sync.get("line_count", 0) or 0)
|
||||
enabled = bool(log_sync.get("enabled", False))
|
||||
mode = ensure_str(log_sync.get("mode")) or "key"
|
||||
status = "standby"
|
||||
status_label = "待命"
|
||||
summary = "当前没有参与检测节点,现场日志观察处于待命状态。"
|
||||
if participating_total > 0 and not enabled:
|
||||
status = "disabled"
|
||||
status_label = "未开启"
|
||||
summary = (
|
||||
f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭,"
|
||||
"海外控制面还看不到节点级现场日志。"
|
||||
)
|
||||
elif participating_total > 0 and (
|
||||
line_count <= 0 or covered_total <= 0 or int(log_sync.get("missing_participating_node_count", len(missing_nodes)) or len(missing_nodes)) > 0
|
||||
):
|
||||
status = "waiting_sample"
|
||||
status_label = "等待样本"
|
||||
summary = (
|
||||
f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total},"
|
||||
f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。"
|
||||
)
|
||||
elif participating_total > 0:
|
||||
status = "ready"
|
||||
status_label = "可下钻"
|
||||
summary = (
|
||||
f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点,"
|
||||
"可以直接下钻节点现场日志。"
|
||||
)
|
||||
|
||||
focus_ref = {}
|
||||
if target_node_code:
|
||||
focus_ref = {
|
||||
"kind": "node_scene_log",
|
||||
"node_code": target_node_code,
|
||||
"mode": mode,
|
||||
"limit": 120 if status == "waiting_sample" else 80,
|
||||
}
|
||||
return {
|
||||
"status": status,
|
||||
"status_label": status_label,
|
||||
"summary": summary,
|
||||
"focus_ref": focus_ref,
|
||||
"target_node_code": target_node_code,
|
||||
"participating_node_count": participating_total,
|
||||
"covered_participating_node_count": covered_total,
|
||||
"line_count": line_count,
|
||||
}
|
||||
|
||||
|
||||
base_url = ensure_str(sys.argv[1]) or "http://127.0.0.1:8100"
|
||||
overview = as_dict(sys.argv[2])
|
||||
nodes = as_dict(sys.argv[3])
|
||||
jobs = as_dict(sys.argv[4])
|
||||
activity = as_dict(sys.argv[5])
|
||||
launchpad = as_dict(sys.argv[6])
|
||||
latest_release = as_dict(sys.argv[7])
|
||||
policy_preview = as_dict(sys.argv[8])
|
||||
|
||||
recommendation = ensure_dict(overview.get("recommendation"))
|
||||
scene_log = normalize_scene_log_observation(overview)
|
||||
managed_nodes = ensure_list(nodes.get("managed_nodes"))
|
||||
node_summary = ensure_dict(nodes.get("summary"))
|
||||
jobs_summary = ensure_dict(jobs.get("summary"))
|
||||
activity_summary = ensure_dict(activity.get("summary"))
|
||||
launchpad_release = ensure_dict(launchpad.get("release"))
|
||||
launchpad_rec = ensure_dict(launchpad.get("top_recommendation"))
|
||||
launchpad_workers = ensure_dict(launchpad.get("worker_rollout_preview"))
|
||||
launchpad_controls = ensure_dict(launchpad.get("control_rollout_preview"))
|
||||
launchpad_status = ensure_dict(launchpad.get("launchpad_status"))
|
||||
latest_release_entry = latest_release if isinstance(latest_release, dict) else {}
|
||||
latest_package_preview = ensure_dict(launchpad.get("latest_package_preview"))
|
||||
|
||||
online_nodes = [
|
||||
node for node in managed_nodes
|
||||
if str(node.get("status") or "").lower() in {"online", "busy"}
|
||||
]
|
||||
attention_nodes = [
|
||||
{
|
||||
"node_code": str(node.get("node_code") or ""),
|
||||
"status": str(node.get("status") or ""),
|
||||
"role": str(node.get("role") or ""),
|
||||
"stack_status_label": str(node.get("stack_status_label") or ""),
|
||||
"recommended_action_code": str(node.get("recommended_action_code") or ""),
|
||||
"recommended_action_label": str(node.get("recommended_action_label") or ""),
|
||||
"primary_issue": str(node.get("primary_issue") or ""),
|
||||
}
|
||||
for node in managed_nodes
|
||||
if str(node.get("status") or "").lower() not in {"online", "busy"}
|
||||
or str(node.get("stack_status") or "").lower() not in {"healthy", "ready", ""}
|
||||
]
|
||||
|
||||
activity_items = ensure_list(activity.get("items"))
|
||||
recent_activity = [
|
||||
{
|
||||
"kind": str(item.get("kind") or ""),
|
||||
"title": str(item.get("title") or ""),
|
||||
"level": str(item.get("level") or ""),
|
||||
"created_at": str(item.get("created_at") or ""),
|
||||
}
|
||||
for item in activity_items[:5]
|
||||
]
|
||||
|
||||
job_items = ensure_list(jobs.get("items"))
|
||||
running_jobs = [
|
||||
{
|
||||
"job_key": str(item.get("job_key") or ""),
|
||||
"status": str(item.get("status") or ""),
|
||||
"title": str(item.get("title") or ""),
|
||||
"progress_text": str(item.get("progress_text") or ""),
|
||||
}
|
||||
for item in job_items
|
||||
if str(item.get("status") or "").lower() in {"running", "queued", "waiting"}
|
||||
][:5]
|
||||
|
||||
recommendation_focus_ref = ensure_dict(recommendation.get("focus_ref"))
|
||||
first_attention = attention_nodes[0] if attention_nodes else {}
|
||||
first_attention_node_code = ensure_str(first_attention.get("node_code"))
|
||||
scene_log_target_node_code = ensure_str(scene_log.get("target_node_code"))
|
||||
latest_release_version = str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or "")
|
||||
release_focus_ref = ensure_dict(launchpad_rec.get("focus_ref"))
|
||||
release_id = str(release_focus_ref.get("release_id") or "")
|
||||
operator_decision = build_operator_decision()
|
||||
primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
|
||||
secondary_command_key = ensure_str(operator_decision.get("secondary_command_key"))
|
||||
|
||||
result = {
|
||||
"overview": {
|
||||
"top_recommendation": {
|
||||
"source": str(recommendation.get("source") or ""),
|
||||
"key": str(recommendation.get("key") or ""),
|
||||
"priority": str(recommendation.get("priority") or ""),
|
||||
"summary": str(recommendation.get("summary") or ""),
|
||||
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
|
||||
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
|
||||
"focus_ref": recommendation.get("focus_ref") or {},
|
||||
"node_codes": list(recommendation.get("node_codes") or []),
|
||||
},
|
||||
"scene_log_observation": {
|
||||
"status": str(scene_log.get("status") or ""),
|
||||
"status_label": str(scene_log.get("status_label") or ""),
|
||||
"summary": str(scene_log.get("summary") or ""),
|
||||
"target_node_code": str(scene_log.get("target_node_code") or ""),
|
||||
"participating_node_count": int(scene_log.get("participating_node_count") or 0),
|
||||
"covered_participating_node_count": int(scene_log.get("covered_participating_node_count") or 0),
|
||||
"line_count": int(scene_log.get("line_count") or 0),
|
||||
"focus_ref": scene_log.get("focus_ref") or {},
|
||||
},
|
||||
},
|
||||
"nodes": {
|
||||
"managed_total": int(node_summary.get("managed_total") or len(managed_nodes)),
|
||||
"online_or_busy_total": len(online_nodes),
|
||||
"offline_total": int(node_summary.get("offline_count") or 0),
|
||||
"attention_total": len(attention_nodes),
|
||||
"attention_nodes": attention_nodes[:8],
|
||||
},
|
||||
"jobs": {
|
||||
"running_total": int(jobs_summary.get("running_count") or 0),
|
||||
"queued_total": int(jobs_summary.get("queued_count") or 0),
|
||||
"failed_total": int(jobs_summary.get("failed_count") or 0),
|
||||
"recent_running_jobs": running_jobs,
|
||||
},
|
||||
"activity_stream": {
|
||||
"total": int(activity_summary.get("total") or len(activity_items)),
|
||||
"error_count": int(activity_summary.get("error_count") or 0),
|
||||
"warning_count": int(activity_summary.get("warning_count") or 0),
|
||||
"running_job_count": int(activity_summary.get("running_job_count") or 0),
|
||||
"recent_items": recent_activity,
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": ensure_str(operator_decision.get("lane")),
|
||||
"priority": ensure_str(operator_decision.get("priority")),
|
||||
"reason_code": ensure_str(operator_decision.get("reason_code")),
|
||||
"title": ensure_str(operator_decision.get("title")),
|
||||
"summary": ensure_str(operator_decision.get("summary")),
|
||||
"next_focus": ensure_dict(operator_decision.get("next_focus")),
|
||||
"primary_command_key": primary_command_key,
|
||||
"secondary_command_key": secondary_command_key,
|
||||
},
|
||||
"recommended_commands": {
|
||||
"stack_diagnosis": "bash domain-api/deploy/multi-region/check_ops_center_stack.sh",
|
||||
"driver_feed": command("driver-feed"),
|
||||
"activity_preview": command("activity-preview"),
|
||||
"driver_focus_preview": command("driver-focus-preview"),
|
||||
"driver_focus_run": command("driver-focus-run"),
|
||||
"top_recommendation_preview": (
|
||||
command("driver-preview", str(recommendation.get("primary_action_code") or ""))
|
||||
if str(recommendation.get("primary_action_code") or "").strip()
|
||||
else ""
|
||||
),
|
||||
"top_recommendation_focus_preview": (
|
||||
command("driver-focus-preview", f"entry:{str(recommendation.get('key') or '')}")
|
||||
if str(recommendation.get("key") or "").strip()
|
||||
else ""
|
||||
),
|
||||
"scene_log": (
|
||||
command("scene-node-log", scene_log_target_node_code, "120", "key")
|
||||
if scene_log_target_node_code
|
||||
else ""
|
||||
),
|
||||
"scene_log_direct": (
|
||||
f"bash domain-api/deploy/multi-region/check_node_scene_log.sh {base_url} {scene_log_target_node_code} 120 key"
|
||||
if scene_log_target_node_code
|
||||
else ""
|
||||
),
|
||||
"node_handover": (
|
||||
command("node-handover", first_attention_node_code)
|
||||
if first_attention_node_code
|
||||
else command("stack-first-gap-handover")
|
||||
),
|
||||
"node_bootstrap_plan": (
|
||||
command("node-bootstrap-plan", first_attention_node_code)
|
||||
if first_attention_node_code
|
||||
else ""
|
||||
),
|
||||
"delivery_queue": (
|
||||
command("delivery-queue", first_attention_node_code)
|
||||
if first_attention_node_code
|
||||
else ""
|
||||
),
|
||||
"release_launchpad": release_command("launchpad"),
|
||||
"release_preview_latest": release_command("preview-latest"),
|
||||
"release_preview_latest_worker": release_command("preview-latest-smart", "worker"),
|
||||
"release_preview_latest_control": release_command("preview-latest-smart", "control"),
|
||||
"release_focus": (
|
||||
command("driver-preview", "focus_release_hub")
|
||||
if str(launchpad_rec.get("action_code") or "").strip()
|
||||
else ""
|
||||
),
|
||||
"release_preview_selected": (
|
||||
release_command("preview-release", release_id, "worker")
|
||||
if release_id
|
||||
else ""
|
||||
),
|
||||
"jobs": command("activity-stream", "kind=ops_job"),
|
||||
},
|
||||
"next_actions": {
|
||||
"primary_command": "",
|
||||
"secondary_command": "",
|
||||
},
|
||||
"release_launchpad": {
|
||||
"status": str(launchpad_status.get("status") or ""),
|
||||
"status_label": str(launchpad_status.get("status_label") or ""),
|
||||
"summary": str(launchpad_status.get("summary") or ""),
|
||||
"recommended_action_code": str(launchpad_rec.get("action_code") or ""),
|
||||
"recommended_action_label": str(launchpad_rec.get("action_label") or ""),
|
||||
"recommended_execution_mode": str(launchpad_rec.get("execution_mode") or ""),
|
||||
"recommended_execution_mode_label": str(launchpad_rec.get("execution_mode_label") or ""),
|
||||
"focus_ref": launchpad_rec.get("focus_ref") or {},
|
||||
"release_version": str(launchpad_release.get("version") or launchpad_release.get("release_version") or ""),
|
||||
"worker_rollout_preview": {
|
||||
"available": bool(launchpad_workers.get("available")),
|
||||
"blocked": bool(launchpad_workers.get("blocked")),
|
||||
"execution_mode": str(launchpad_workers.get("execution_mode") or ""),
|
||||
"execution_mode_label": str(launchpad_workers.get("execution_mode_label") or ""),
|
||||
"target_node_codes": list(launchpad_workers.get("target_node_codes") or []),
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"available": bool(launchpad_controls.get("available")),
|
||||
"blocked": bool(launchpad_controls.get("blocked")),
|
||||
"execution_mode": str(launchpad_controls.get("execution_mode") or ""),
|
||||
"execution_mode_label": str(launchpad_controls.get("execution_mode_label") or ""),
|
||||
"target_node_codes": list(launchpad_controls.get("target_node_codes") or []),
|
||||
},
|
||||
},
|
||||
"latest_release": {
|
||||
"version": str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or ""),
|
||||
"status": ensure_str(latest_release_entry.get("status")),
|
||||
"status_label": ensure_str(latest_release_entry.get("status_label")),
|
||||
"summary": ensure_str(latest_release_entry.get("summary") or latest_release_entry.get("summary_text")),
|
||||
},
|
||||
"policy_preview_sample": {
|
||||
"execution_mode": str(policy_preview.get("execution_mode") or ""),
|
||||
"risk_level": str(policy_preview.get("risk_level") or ""),
|
||||
"approval_required": bool(policy_preview.get("approval_required")),
|
||||
"blocked": bool(policy_preview.get("blocked")),
|
||||
"blocking_reasons": ensure_list(policy_preview.get("blocking_reasons"))[:5],
|
||||
"warnings": ensure_list(policy_preview.get("warnings"))[:5],
|
||||
},
|
||||
"focus_refs": {
|
||||
"top_recommendation": recommendation_focus_ref,
|
||||
"scene_log": ensure_dict(scene_log.get("focus_ref")),
|
||||
"release_hub": release_focus_ref,
|
||||
},
|
||||
"rollout_targets": {
|
||||
"worker": list(launchpad_workers.get("target_node_codes") or []),
|
||||
"control": list(launchpad_controls.get("target_node_codes") or []),
|
||||
},
|
||||
"package_preview": {
|
||||
"available": bool(latest_package_preview.get("release")),
|
||||
"preview_source": ensure_str(latest_package_preview.get("preview_source")),
|
||||
"message": ensure_str(latest_package_preview.get("message")),
|
||||
},
|
||||
}
|
||||
|
||||
result["next_actions"]["primary_command"] = ensure_str(
|
||||
ensure_dict(result.get("recommended_commands")).get(primary_command_key)
|
||||
)
|
||||
result["next_actions"]["secondary_command"] = ensure_str(
|
||||
ensure_dict(result.get("recommended_commands")).get(secondary_command_key)
|
||||
)
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
373
domain-api/deploy/multi-region/check_release_hub.sh
Normal file
373
domain-api/deploy/multi-region/check_release_hub.sh
Normal file
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
INPUT_RELEASE_ID="${2:-}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
|
||||
|
||||
curl_json() {
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"$1"
|
||||
}
|
||||
|
||||
curl_post_json() {
|
||||
local url="$1"
|
||||
local body="$2"
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
-X POST "${url}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "${body}"
|
||||
}
|
||||
|
||||
launchpad="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")"
|
||||
latest_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")"
|
||||
action_templates="$(curl_json "${BASE_URL}/api/v1/ops/action-templates")"
|
||||
cluster_json="$(curl_json "${BASE_URL}/api/v1/runtime/cluster")"
|
||||
release_id="${INPUT_RELEASE_ID}"
|
||||
|
||||
if [[ -z "${release_id}" ]]; then
|
||||
release_id="$(python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
data = payload.get("data") or {}
|
||||
value = data.get("id") or 0
|
||||
print(value)
|
||||
' "${latest_release}")"
|
||||
fi
|
||||
|
||||
selected_release="${latest_release}"
|
||||
if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then
|
||||
selected_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}")"
|
||||
fi
|
||||
|
||||
build_release_preview_payload() {
|
||||
local preview_mode="$1"
|
||||
python3 - <<'PY' "${selected_release}" "${cluster_json}" "${preview_mode}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
release_payload = json.loads(sys.argv[1]).get("data") or {}
|
||||
cluster_payload = json.loads(sys.argv[2]).get("data") or {}
|
||||
preview_mode = str(sys.argv[3] or "").strip().lower()
|
||||
|
||||
nodes = list(cluster_payload.get("nodes") or [])
|
||||
|
||||
def is_online(node):
|
||||
return str(node.get("status") or "").strip() in {"online", "busy"}
|
||||
|
||||
control_nodes = [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node) and str(node.get("role") or "").strip() == "control"
|
||||
]
|
||||
worker_nodes = [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node)
|
||||
and str(node.get("role") or "").strip() == "worker"
|
||||
and bool(node.get("is_effective_worker", False))
|
||||
]
|
||||
online_nodes = [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node)
|
||||
]
|
||||
|
||||
payload = {
|
||||
"release_version": str(release_payload.get("release_version") or "").strip(),
|
||||
"artifact_url": str(release_payload.get("artifact_url") or "").strip(),
|
||||
"checksum": str(release_payload.get("checksum") or "").strip(),
|
||||
"install_root": "/opt/domaincheck",
|
||||
"health_check_timeout_seconds": 10,
|
||||
"health_check_retries": 2,
|
||||
"health_check_interval_seconds": 2,
|
||||
"rollback_on_failure": True,
|
||||
"switch_current": True,
|
||||
}
|
||||
|
||||
if preview_mode == "control":
|
||||
template_key = "deploy.release.control"
|
||||
target_node_codes = control_nodes
|
||||
payload.update(
|
||||
{
|
||||
"restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
"health_check_urls": ["http://127.0.0.1:8100/health"],
|
||||
"health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
}
|
||||
)
|
||||
elif preview_mode == "worker":
|
||||
template_key = "deploy.release.worker"
|
||||
target_node_codes = worker_nodes
|
||||
payload.update(
|
||||
{
|
||||
"restart_services": ["domaincheck-worker"],
|
||||
"health_check_urls": [],
|
||||
"health_check_services": ["domaincheck-worker"],
|
||||
}
|
||||
)
|
||||
else:
|
||||
template_key = "deploy.release.custom"
|
||||
target_node_codes = (control_nodes[:1] + worker_nodes[:2]) or online_nodes[:3]
|
||||
payload.update(
|
||||
{
|
||||
"restart_services": [],
|
||||
"health_check_urls": [],
|
||||
"health_check_services": [],
|
||||
}
|
||||
)
|
||||
|
||||
request_payload = {
|
||||
"template_key": template_key,
|
||||
"action": "deploy.release",
|
||||
"target_type": "batch",
|
||||
"execution_mode": "remote-agent",
|
||||
"target_node_codes": [item for item in target_node_codes if item],
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
print(json.dumps(request_payload, ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
print_preview_request_summary() {
|
||||
local preview_payload="$1"
|
||||
python3 - <<'PY' "${preview_payload}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
body = payload.get("payload") or {}
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"template_key": payload.get("template_key") or "",
|
||||
"action": payload.get("action") or "",
|
||||
"execution_mode": payload.get("execution_mode") or "",
|
||||
"target_node_codes": payload.get("target_node_codes") or [],
|
||||
"release_version": body.get("release_version") or "",
|
||||
"artifact_url_present": bool(str(body.get("artifact_url") or "").strip()),
|
||||
"checksum_present": bool(str(body.get("checksum") or "").strip()),
|
||||
"restart_services": body.get("restart_services") or [],
|
||||
"health_check_urls": body.get("health_check_urls") or [],
|
||||
"health_check_services": body.get("health_check_services") or [],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
PY
|
||||
}
|
||||
|
||||
run_release_preview() {
|
||||
local step_no="$1"
|
||||
local preview_mode="$2"
|
||||
local section_title="$3"
|
||||
local preview_payload
|
||||
preview_payload="$(build_release_preview_payload "${preview_mode}")"
|
||||
|
||||
echo "[${step_no}/10] ${section_title}"
|
||||
print_preview_request_summary "${preview_payload}"
|
||||
echo
|
||||
curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" "${preview_payload}"
|
||||
echo
|
||||
echo
|
||||
}
|
||||
|
||||
echo "[1/11] release launchpad"
|
||||
printf '%s\n\n' "${launchpad}"
|
||||
|
||||
echo "[2/11] latest release"
|
||||
printf '%s\n\n' "${selected_release}"
|
||||
|
||||
echo "[3/11] recent releases"
|
||||
curl_json "${BASE_URL}/api/v1/ops/releases?limit=8"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[4/11] release deploy templates"
|
||||
python3 - <<'PY' "${action_templates}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
groups = (payload.get("data") or {}).get("groups") or []
|
||||
release_group = next((item for item in groups if str(item.get("group_key") or "") == "release"), {})
|
||||
items = list(release_group.get("items") or [])
|
||||
summary = {
|
||||
"group_key": release_group.get("group_key") or "",
|
||||
"group_title": release_group.get("group_title") or "",
|
||||
"template_keys": [str(item.get("key") or "") for item in items],
|
||||
"field_types": sorted(
|
||||
{
|
||||
str(field.get("type") or "")
|
||||
for item in items
|
||||
for field in list(item.get("fields") or [])
|
||||
if str(field.get("type") or "")
|
||||
}
|
||||
),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
|
||||
echo "[5/11] online release target summary"
|
||||
python3 - <<'PY' "${cluster_json}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1]).get("data") or {}
|
||||
nodes = list(payload.get("nodes") or [])
|
||||
|
||||
def is_online(node):
|
||||
return str(node.get("status") or "").strip() in {"online", "busy"}
|
||||
|
||||
summary = {
|
||||
"online_control_nodes": [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node) and str(node.get("role") or "").strip() == "control"
|
||||
],
|
||||
"online_worker_nodes": [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False))
|
||||
],
|
||||
"all_online_nodes": [
|
||||
{
|
||||
"node_code": str(node.get("node_code") or "").strip(),
|
||||
"role": str(node.get("role") or "").strip(),
|
||||
"status": str(node.get("status") or "").strip(),
|
||||
"is_effective_worker": bool(node.get("is_effective_worker", False)),
|
||||
}
|
||||
for node in nodes
|
||||
if is_online(node)
|
||||
],
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
echo
|
||||
|
||||
run_release_preview "6" "control" "deploy.release.control preview"
|
||||
run_release_preview "7" "worker" "deploy.release.worker preview"
|
||||
run_release_preview "8" "custom" "deploy.release.custom preview"
|
||||
|
||||
rollout_list='{"code":0,"message":"no release selected","data":{"rollouts":[]}}'
|
||||
rollout_id="0"
|
||||
if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then
|
||||
rollout_list="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}/rollouts?limit=10")"
|
||||
rollout_id="$(python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
rollouts = (payload.get("data") or {}).get("rollouts") or []
|
||||
print((rollouts[0] or {}).get("id") or 0 if rollouts else 0)
|
||||
' "${rollout_list}")"
|
||||
fi
|
||||
|
||||
echo "[9/11] rollout list for release ${release_id:-0}"
|
||||
printf '%s\n\n' "${rollout_list}"
|
||||
|
||||
echo "[10/11] primary rollout detail"
|
||||
if [[ "${rollout_id}" != "0" ]]; then
|
||||
curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}"
|
||||
else
|
||||
echo '{"code":0,"message":"no rollout","data":{}}'
|
||||
fi
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[11/12] primary rollout jobs"
|
||||
if [[ "${rollout_id}" != "0" ]]; then
|
||||
curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}/jobs?limit=50"
|
||||
else
|
||||
echo '{"code":0,"message":"no rollout","data":{"jobs":[]}}'
|
||||
fi
|
||||
echo
|
||||
|
||||
echo
|
||||
echo "[12/12] condensed summary"
|
||||
python3 - <<'PY' "${launchpad}" "${selected_release}" "${cluster_json}" "${rollout_list}" "${rollout_id}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
launchpad_payload = json.loads(sys.argv[1] or "{}").get("data") or {}
|
||||
selected_release = json.loads(sys.argv[2] or "{}").get("data") or {}
|
||||
cluster_payload = json.loads(sys.argv[3] or "{}").get("data") or {}
|
||||
rollout_list_payload = json.loads(sys.argv[4] or "{}").get("data") or {}
|
||||
rollout_id = int(sys.argv[5] or 0)
|
||||
|
||||
launchpad_status = dict(launchpad_payload.get("launchpad_status") or {})
|
||||
worker_preview = dict(launchpad_payload.get("worker_rollout_preview") or {})
|
||||
worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {})
|
||||
control_preview = dict(launchpad_payload.get("control_rollout_preview") or {})
|
||||
control_smart_rollout = dict(control_preview.get("smart_rollout") or {})
|
||||
nodes = list(cluster_payload.get("nodes") or [])
|
||||
rollouts = list(rollout_list_payload.get("rollouts") or [])
|
||||
|
||||
def is_online(node):
|
||||
return str(node.get("status") or "").strip() in {"online", "busy"}
|
||||
|
||||
summary = {
|
||||
"release": {
|
||||
"id": int(selected_release.get("id") or 0),
|
||||
"release_version": str(selected_release.get("release_version") or ""),
|
||||
"channel": str(selected_release.get("channel") or ""),
|
||||
"status": str(selected_release.get("status") or ""),
|
||||
},
|
||||
"launchpad": {
|
||||
"status": str(launchpad_status.get("status") or ""),
|
||||
"status_label": str(launchpad_status.get("status_label") or ""),
|
||||
"summary": str(launchpad_status.get("summary") or ""),
|
||||
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
|
||||
"recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""),
|
||||
"recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""),
|
||||
"recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""),
|
||||
"onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
|
||||
"recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""),
|
||||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"available": bool(worker_smart_rollout.get("available", False)),
|
||||
"blocked": bool(worker_preview.get("blocked", False)),
|
||||
"requires_confirmation": bool(worker_preview.get("requires_confirmation", False)),
|
||||
"execution_mode": str(worker_smart_rollout.get("execution_mode") or ""),
|
||||
"target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []),
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"available": bool(control_smart_rollout.get("available", False)),
|
||||
"blocked": bool(control_preview.get("blocked", False)),
|
||||
"requires_confirmation": bool(control_preview.get("requires_confirmation", False)),
|
||||
"execution_mode": str(control_smart_rollout.get("execution_mode") or ""),
|
||||
"target_node_codes": list(control_smart_rollout.get("target_node_codes") or []),
|
||||
},
|
||||
"cluster": {
|
||||
"online_control_nodes": [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node) and str(node.get("role") or "").strip() == "control"
|
||||
],
|
||||
"online_worker_nodes": [
|
||||
str(node.get("node_code") or "").strip()
|
||||
for node in nodes
|
||||
if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False))
|
||||
],
|
||||
},
|
||||
"rollouts": {
|
||||
"count": len(rollouts),
|
||||
"selected_rollout_id": rollout_id,
|
||||
},
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
echo
|
||||
@@ -4,15 +4,24 @@ set -euo pipefail
|
||||
MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}"
|
||||
OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
|
||||
|
||||
curl_json() {
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"$1"
|
||||
}
|
||||
|
||||
echo "[1/5] mainland runtime cluster"
|
||||
MAINLAND_CLUSTER="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
|
||||
MAINLAND_CLUSTER="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")"
|
||||
echo "${MAINLAND_CLUSTER}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[2/5] mainland runtime readiness"
|
||||
MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
|
||||
MAINLAND_READINESS="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")"
|
||||
echo "${MAINLAND_READINESS}"
|
||||
echo
|
||||
echo
|
||||
@@ -79,7 +88,7 @@ echo
|
||||
echo
|
||||
|
||||
echo "[4/5] overseas runtime cluster"
|
||||
OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
|
||||
OVERSEAS_CLUSTER="$(curl_json "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")"
|
||||
echo "${OVERSEAS_CLUSTER}"
|
||||
echo
|
||||
echo
|
||||
|
||||
@@ -4,23 +4,44 @@ set -euo pipefail
|
||||
DB_NAME="${DB_NAME:-domain}"
|
||||
DB_USER="${DB_USER:-postgres}"
|
||||
PSQL_BIN="${PSQL_BIN:-/www/server/pgsql/bin/psql}"
|
||||
API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:8100}}"
|
||||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
|
||||
|
||||
echo "[1/3] cluster nodes"
|
||||
curl_json() {
|
||||
curl -fsS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"$1"
|
||||
}
|
||||
|
||||
RUNTIME_STATUS_JSON="$(curl_json "${API_BASE_URL}/api/v1/runtime/status")"
|
||||
|
||||
echo "[1/4] cluster nodes"
|
||||
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
|
||||
select
|
||||
node_code || '|' ||
|
||||
role || '|' ||
|
||||
status || '|' ||
|
||||
coalesce(current_load, 0) || '|' ||
|
||||
coalesce(metadata_json->>'job_items_claimed', '0') || '|' ||
|
||||
coalesce(metadata_json->>'job_items_running', '0') || '|' ||
|
||||
coalesce(metadata_json->>'job_items_completed', '0') || '|' ||
|
||||
to_char(last_heartbeat_at, 'YYYY-MM-DD HH24:MI:SS') || '|' ||
|
||||
coalesce(metadata_json->>'worker_online', '') || '|' ||
|
||||
coalesce(metadata_json->>'detect_participating', '')
|
||||
(
|
||||
case
|
||||
when coalesce(current_load, 0) > 0 then 'true'
|
||||
when coalesce(metadata_json->>'detect_participating', '') in ('true', 'false') then metadata_json->>'detect_participating'
|
||||
else ''
|
||||
end
|
||||
)
|
||||
from detect_worker_nodes
|
||||
order by node_code;
|
||||
"
|
||||
echo
|
||||
|
||||
echo "[2/3] active job distribution"
|
||||
echo "[2/4] active job distribution"
|
||||
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc "
|
||||
select
|
||||
coalesce(claimed_by, '') || '|' ||
|
||||
@@ -39,59 +60,43 @@ order by claimed_by, status;
|
||||
"
|
||||
echo
|
||||
|
||||
echo "[3/3] condensed summary"
|
||||
"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -At <<'SQL'
|
||||
with latest_job as (
|
||||
select id, job_code, status
|
||||
from detect_jobs
|
||||
where status in ('pending','running')
|
||||
order by id desc
|
||||
limit 1
|
||||
),
|
||||
node_stats as (
|
||||
select
|
||||
node_code,
|
||||
role,
|
||||
status,
|
||||
coalesce(metadata_json->>'worker_online', '') as worker_online,
|
||||
coalesce(metadata_json->>'detect_participating', '') as detect_participating
|
||||
from detect_worker_nodes
|
||||
),
|
||||
item_stats as (
|
||||
select
|
||||
coalesce(claimed_by, '') as claimed_by,
|
||||
status,
|
||||
count(*) as cnt
|
||||
from detect_job_items
|
||||
where job_id = (select id from latest_job)
|
||||
group by claimed_by, status
|
||||
)
|
||||
select json_build_object(
|
||||
'job', (
|
||||
select json_build_object(
|
||||
'job_code', coalesce(job_code, ''),
|
||||
'status', coalesce(status, '')
|
||||
)
|
||||
from latest_job
|
||||
),
|
||||
'online_nodes', (
|
||||
select coalesce(json_agg(json_build_object(
|
||||
'node_code', node_code,
|
||||
'role', role,
|
||||
'status', status,
|
||||
'worker_online', worker_online,
|
||||
'detect_participating', detect_participating
|
||||
) order by node_code), '[]'::json)
|
||||
from node_stats
|
||||
where status in ('online', 'busy')
|
||||
),
|
||||
'claimed_distribution', (
|
||||
select coalesce(json_agg(json_build_object(
|
||||
'claimed_by', claimed_by,
|
||||
'status', status,
|
||||
'count', cnt
|
||||
) order by claimed_by, status), '[]'::json)
|
||||
from item_stats
|
||||
)
|
||||
);
|
||||
SQL
|
||||
echo "[3/4] runtime participation snapshot"
|
||||
printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c '
|
||||
import json, sys
|
||||
payload = json.load(sys.stdin).get("data", {})
|
||||
detect = payload.get("detect", {})
|
||||
print(json.dumps({
|
||||
"node": payload.get("node", {}),
|
||||
"participation_summary": detect.get("participation_summary", {}),
|
||||
"participating_nodes": detect.get("participating_nodes", []),
|
||||
"non_participating_nodes": detect.get("non_participating_nodes", detect.get("standby_nodes", [])),
|
||||
"log_sync": detect.get("log_sync", {}),
|
||||
}, ensure_ascii=False, indent=2))
|
||||
'
|
||||
echo
|
||||
|
||||
echo "[4/4] condensed summary"
|
||||
printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c '
|
||||
import json, sys
|
||||
payload = json.load(sys.stdin).get("data", {})
|
||||
detect = payload.get("detect", {})
|
||||
summary = detect.get("participation_summary", {}) or {}
|
||||
print(json.dumps({
|
||||
"node": payload.get("node", {}),
|
||||
"active_job": {
|
||||
"job_code": (detect.get("active_job") or {}).get("job_code", ""),
|
||||
"status": (detect.get("active_job") or {}).get("status", ""),
|
||||
},
|
||||
"participation_summary": summary,
|
||||
"dispatch_active_nodes": summary.get("dispatch_active_node_codes", []),
|
||||
"recent_only_nodes": summary.get("recent_only_node_codes", []),
|
||||
"non_participating_nodes": summary.get("non_participating_node_codes", []),
|
||||
"log_sync": {
|
||||
"enabled": bool((detect.get("log_sync") or {}).get("enabled", False)),
|
||||
"mode": (detect.get("log_sync") or {}).get("mode", ""),
|
||||
"line_count": (detect.get("log_sync") or {}).get("line_count", 0),
|
||||
"source_nodes": (detect.get("log_sync") or {}).get("source_nodes", []),
|
||||
"last_at": (detect.get("log_sync") or {}).get("last_at", ""),
|
||||
},
|
||||
}, ensure_ascii=False, indent=2))
|
||||
'
|
||||
|
||||
447
domain-api/deploy/multi-region/drive_ops_action.sh
Executable file
447
domain-api/deploy/multi-region/drive_ops_action.sh
Executable file
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib_ops_center.sh"
|
||||
ops_center_load_config "${SCRIPT_DIR}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
usage:
|
||||
bash domain-api/deploy/multi-region/drive_ops_action.sh <base_url> <preview|execute> <action_code> [action_payload_json|@payload.json] [requested_by]
|
||||
bash domain-api/deploy/multi-region/drive_ops_action.sh <base_url> <action_code> [action_payload_json|@payload.json] [requested_by]
|
||||
|
||||
examples:
|
||||
bash domain-api/deploy/multi-region/drive_ops_action.sh \
|
||||
http://127.0.0.1:8100 \
|
||||
preview \
|
||||
create_release_rollout_worker \
|
||||
'{"release_id":12}'
|
||||
|
||||
bash domain-api/deploy/multi-region/drive_ops_action.sh \
|
||||
http://127.0.0.1:8100 \
|
||||
create_release_rollout_worker \
|
||||
'{"release_id":12}'
|
||||
|
||||
NODE_CODES="mainland-worker-01,mainland-worker-02" \
|
||||
bash domain-api/deploy/multi-region/drive_ops_action.sh \
|
||||
http://127.0.0.1:8100 \
|
||||
run_standard_inspection \
|
||||
'{}'
|
||||
|
||||
notes:
|
||||
- preview/execute 双模式共用同一份 action payload 和 node_codes
|
||||
- 旧用法仍默认按 execute 处理
|
||||
- NODE_CODES can be passed through env as comma/newline separated node_code list
|
||||
- action_payload_json can also use @/path/to/file.json
|
||||
- preview metadata can be passed through env:
|
||||
SOURCE_LABEL / ACTION_TITLE / ACTION_SUMMARY / ACTION_REASON / ACTION_META_TEXT / PRIMARY_LABEL
|
||||
EOF
|
||||
}
|
||||
|
||||
BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}"
|
||||
SECOND_ARG="${2:-}"
|
||||
THIRD_ARG="${3:-}"
|
||||
FOURTH_ARG="${4:-}"
|
||||
FIFTH_ARG="${5:-}"
|
||||
|
||||
if [[ "${SECOND_ARG}" == "preview" || "${SECOND_ARG}" == "execute" ]]; then
|
||||
OPERATION="${SECOND_ARG}"
|
||||
ACTION_CODE="${THIRD_ARG:-}"
|
||||
RAW_ACTION_PAYLOAD="${FOURTH_ARG:-}"
|
||||
REQUESTED_BY="${FIFTH_ARG:-$(ops_center_resolve_requested_by)}"
|
||||
else
|
||||
OPERATION="execute"
|
||||
ACTION_CODE="${SECOND_ARG:-}"
|
||||
RAW_ACTION_PAYLOAD="${THIRD_ARG:-}"
|
||||
REQUESTED_BY="${FOURTH_ARG:-$(ops_center_resolve_requested_by)}"
|
||||
fi
|
||||
|
||||
if [[ -z "${RAW_ACTION_PAYLOAD}" ]]; then
|
||||
RAW_ACTION_PAYLOAD='{}'
|
||||
fi
|
||||
|
||||
NODE_CODES_INPUT="${NODE_CODES:-}"
|
||||
SOURCE_LABEL="${SOURCE_LABEL:-cli/drive_ops_action}"
|
||||
ACTION_TITLE="${ACTION_TITLE:-}"
|
||||
ACTION_SUMMARY="${ACTION_SUMMARY:-}"
|
||||
ACTION_REASON="${ACTION_REASON:-}"
|
||||
ACTION_META_TEXT="${ACTION_META_TEXT:-}"
|
||||
PRIMARY_LABEL="${PRIMARY_LABEL:-}"
|
||||
|
||||
print_json_or_raw() {
|
||||
local raw_text="${1:-}"
|
||||
python3 - "${raw_text}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
raw_text = str(sys.argv[1] or "")
|
||||
try:
|
||||
payload = json.loads(raw_text)
|
||||
except Exception:
|
||||
print(raw_text)
|
||||
else:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
}
|
||||
|
||||
request_with_status() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
local body="${3:-}"
|
||||
if [[ "${method}" == "POST" ]]; then
|
||||
curl -sS -X POST "${url}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "${body}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}'
|
||||
else
|
||||
curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}'
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -z "${ACTION_CODE}" ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${OPERATION}" != "preview" && "${OPERATION}" != "execute" ]]; then
|
||||
echo "operation must be preview or execute" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ACTION_PAYLOAD_JSON="${RAW_ACTION_PAYLOAD}"
|
||||
if [[ "${ACTION_PAYLOAD_JSON}" == @* ]]; then
|
||||
ACTION_PAYLOAD_PATH="${ACTION_PAYLOAD_JSON#@}"
|
||||
if [[ ! -f "${ACTION_PAYLOAD_PATH}" ]]; then
|
||||
echo "payload file not found: ${ACTION_PAYLOAD_PATH}" >&2
|
||||
exit 1
|
||||
fi
|
||||
ACTION_PAYLOAD_JSON="$(cat "${ACTION_PAYLOAD_PATH}")"
|
||||
fi
|
||||
|
||||
if [[ -z "${NODE_CODES_INPUT}" ]]; then
|
||||
NODE_CODES_INPUT="$(python3 - <<'PY' "${BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}"
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
def normalize_codes(*groups):
|
||||
result = []
|
||||
for group in groups:
|
||||
for item in list(group or []):
|
||||
node_code = str(item or "").strip()
|
||||
if node_code and node_code not in result:
|
||||
result.append(node_code)
|
||||
return result
|
||||
|
||||
|
||||
base_url = str(sys.argv[1] or "").rstrip("/")
|
||||
action_code = str(sys.argv[2] or "").strip()
|
||||
payload_raw = str(sys.argv[3] or "").strip() or "{}"
|
||||
|
||||
try:
|
||||
payload = json.loads(payload_raw)
|
||||
except Exception:
|
||||
payload = {}
|
||||
if not isinstance(payload, dict):
|
||||
payload = {}
|
||||
|
||||
payload_node_codes = normalize_codes(payload.get("node_codes") or [])
|
||||
if payload_node_codes:
|
||||
print(",".join(payload_node_codes))
|
||||
raise SystemExit(0)
|
||||
|
||||
supported_actions = {
|
||||
"open_worker_logs_participating",
|
||||
"run_inspection_participating",
|
||||
"open_worker_logs_standby",
|
||||
"run_inspection_standby",
|
||||
"open_worker_logs",
|
||||
"run_standard_inspection",
|
||||
}
|
||||
if action_code not in supported_actions or not base_url:
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(f"{base_url}/api/v1/ops/overview", timeout=20) as response:
|
||||
overview_payload = json.loads(response.read().decode("utf-8"))
|
||||
overview = dict((overview_payload or {}).get("data") or {})
|
||||
except Exception:
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
|
||||
execution_scene = dict(overview.get("execution_scene") or {})
|
||||
participation_summary = dict(execution_scene.get("participation_summary") or {})
|
||||
log_sync = dict(execution_scene.get("log_sync") or {})
|
||||
|
||||
dispatch_active = normalize_codes(participation_summary.get("dispatch_active_node_codes") or [])
|
||||
recent_only = normalize_codes(participation_summary.get("recent_only_node_codes") or [])
|
||||
standby = normalize_codes(
|
||||
participation_summary.get("standby_node_codes") or [],
|
||||
participation_summary.get("non_participating_node_codes") or [],
|
||||
)
|
||||
missing_logs = normalize_codes(log_sync.get("missing_participating_nodes") or [])
|
||||
|
||||
resolved_node_codes = []
|
||||
if action_code in {"open_worker_logs_participating", "run_inspection_participating"}:
|
||||
resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only)
|
||||
elif action_code in {"open_worker_logs_standby", "run_inspection_standby"}:
|
||||
resolved_node_codes = normalize_codes(standby)
|
||||
elif action_code == "open_worker_logs":
|
||||
resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only, standby)
|
||||
elif action_code == "run_standard_inspection":
|
||||
resolved_node_codes = normalize_codes(dispatch_active, recent_only, standby)
|
||||
|
||||
print(",".join(resolved_node_codes))
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
action_code = str(sys.argv[1] or "").strip()
|
||||
payload_raw = str(sys.argv[2] or "").strip() or "{}"
|
||||
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action"
|
||||
node_codes_raw = str(sys.argv[4] or "")
|
||||
|
||||
try:
|
||||
action_payload = json.loads(payload_raw)
|
||||
except Exception as exc: # pragma: no cover - cli validation
|
||||
raise SystemExit(f"invalid action payload json: {exc}")
|
||||
|
||||
if not isinstance(action_payload, dict):
|
||||
raise SystemExit("action payload must be a json object")
|
||||
|
||||
node_codes = []
|
||||
seen = set()
|
||||
for item in node_codes_raw.replace("\n", ",").split(","):
|
||||
value = str(item or "").strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
node_codes.append(value)
|
||||
|
||||
request_payload = {
|
||||
"action_code": action_code,
|
||||
"node_codes": node_codes,
|
||||
"action_payload": action_payload,
|
||||
"requested_by": requested_by,
|
||||
}
|
||||
print(json.dumps(request_payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
|
||||
PREVIEW_REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}" "${SOURCE_LABEL}" "${ACTION_TITLE}" "${ACTION_SUMMARY}" "${ACTION_REASON}" "${ACTION_META_TEXT}" "${PRIMARY_LABEL}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
action_code = str(sys.argv[1] or "").strip()
|
||||
payload_raw = str(sys.argv[2] or "").strip() or "{}"
|
||||
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action"
|
||||
node_codes_raw = str(sys.argv[4] or "")
|
||||
source_label = str(sys.argv[5] or "").strip() or "cli/drive_ops_action"
|
||||
action_title = str(sys.argv[6] or "").strip()
|
||||
action_summary = str(sys.argv[7] or "").strip()
|
||||
action_reason = str(sys.argv[8] or "").strip()
|
||||
action_meta_text = str(sys.argv[9] or "").strip()
|
||||
primary_label = str(sys.argv[10] or "").strip()
|
||||
|
||||
try:
|
||||
action_payload = json.loads(payload_raw)
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"invalid action payload json: {exc}")
|
||||
|
||||
if not isinstance(action_payload, dict):
|
||||
raise SystemExit("action payload must be a json object")
|
||||
|
||||
node_codes = []
|
||||
seen = set()
|
||||
for item in node_codes_raw.replace("\n", ",").split(","):
|
||||
value = str(item or "").strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
node_codes.append(value)
|
||||
|
||||
request_payload = {
|
||||
"source_label": source_label,
|
||||
"title": action_title or primary_label or action_code,
|
||||
"summary": action_summary,
|
||||
"reason": action_reason,
|
||||
"meta_text": action_meta_text,
|
||||
"executor_kind": "driver_action",
|
||||
"primary_label": primary_label or action_code,
|
||||
"primary_action_code": action_code,
|
||||
"primary_node_codes": node_codes,
|
||||
"primary_action_payload": action_payload,
|
||||
"requested_by": requested_by,
|
||||
}
|
||||
print(json.dumps(request_payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
|
||||
if [[ "${OPERATION}" == "preview" ]]; then
|
||||
echo "[1/3] driver action preview request"
|
||||
print_json_or_raw "${PREVIEW_REQUEST_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/preview" "${PREVIEW_REQUEST_JSON}")"
|
||||
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
|
||||
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
|
||||
|
||||
echo "[2/3] raw response"
|
||||
printf '%s\n' "http_status=${HTTP_STATUS}"
|
||||
print_json_or_raw "${RESPONSE_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[3/3] condensed summary"
|
||||
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
http_status = int(sys.argv[2] or 0)
|
||||
data = payload.get("data") or {}
|
||||
primary = data.get("primary") or {}
|
||||
secondary = data.get("secondary") or {}
|
||||
|
||||
raw_code = payload.get("code", 1)
|
||||
if raw_code in (0, "0", False):
|
||||
normalized_code = 0
|
||||
else:
|
||||
normalized_code = int(raw_code or 1)
|
||||
|
||||
summary = {
|
||||
"http_status": http_status,
|
||||
"code": normalized_code,
|
||||
"message": str(payload.get("message") or ""),
|
||||
"contract_key": str(data.get("contract_key") or ""),
|
||||
"contract_version": str(data.get("contract_version") or ""),
|
||||
"preview_endpoint": str(data.get("preview_endpoint") or ""),
|
||||
"source_label": str(data.get("source_label") or ""),
|
||||
"title": str(data.get("title") or ""),
|
||||
"executor_kind": str(data.get("executor_kind") or ""),
|
||||
"execution_chain": str(data.get("execution_chain") or ""),
|
||||
"primary": {
|
||||
"label": str(primary.get("label") or ""),
|
||||
"action_code": str(primary.get("action_code") or ""),
|
||||
"target_method": str(primary.get("target_method") or ""),
|
||||
"target_api": str(primary.get("target_api") or ""),
|
||||
"recommendation_label": str(primary.get("recommendation_label") or ""),
|
||||
"automation_level_label": str(primary.get("automation_level_label") or ""),
|
||||
"risk_level_label": str(primary.get("risk_level_label") or ""),
|
||||
"node_codes": list(primary.get("node_codes") or []),
|
||||
"request_payload": primary.get("request_payload") or {},
|
||||
},
|
||||
}
|
||||
|
||||
if secondary:
|
||||
summary["secondary"] = {
|
||||
"label": str(secondary.get("label") or ""),
|
||||
"action_code": str(secondary.get("action_code") or ""),
|
||||
"target_method": str(secondary.get("target_method") or ""),
|
||||
"target_api": str(secondary.get("target_api") or ""),
|
||||
"recommendation_label": str(secondary.get("recommendation_label") or ""),
|
||||
"automation_level_label": str(secondary.get("automation_level_label") or ""),
|
||||
"risk_level_label": str(secondary.get("risk_level_label") or ""),
|
||||
"node_codes": list(secondary.get("node_codes") or []),
|
||||
"request_payload": secondary.get("request_payload") or {},
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
else
|
||||
echo "[1/3] driver action request"
|
||||
print_json_or_raw "${REQUEST_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/execute" "${REQUEST_JSON}")"
|
||||
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
|
||||
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
|
||||
|
||||
echo "[2/3] raw response"
|
||||
printf '%s\n' "http_status=${HTTP_STATUS}"
|
||||
print_json_or_raw "${RESPONSE_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[3/3] condensed summary"
|
||||
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
payload = json.loads(sys.argv[1])
|
||||
http_status = int(sys.argv[2] or 0)
|
||||
data = payload.get("data") or {}
|
||||
smart_release_result = data.get("smart_release_result") or {}
|
||||
ui_intent = data.get("ui_intent") or {}
|
||||
runtime_settings = data.get("runtime_settings") or {}
|
||||
|
||||
raw_code = payload.get("code", 1)
|
||||
if raw_code in (0, "0", False):
|
||||
normalized_code = 0
|
||||
else:
|
||||
normalized_code = int(raw_code or 1)
|
||||
|
||||
summary = {
|
||||
"http_status": http_status,
|
||||
"code": normalized_code,
|
||||
"message": str(payload.get("message") or ""),
|
||||
"handled": bool(data.get("handled", False)),
|
||||
"action_code": str(data.get("action_code") or ""),
|
||||
"mode": str(data.get("mode") or ""),
|
||||
}
|
||||
|
||||
if smart_release_result:
|
||||
release = smart_release_result.get("release") or {}
|
||||
rollout = smart_release_result.get("rollout") or {}
|
||||
smart_plan = smart_release_result.get("smart_rollout") or {}
|
||||
preview = smart_release_result.get("preview") or {}
|
||||
summary["smart_release_result"] = {
|
||||
"release_id": int(release.get("id") or 0),
|
||||
"release_version": str(release.get("release_version") or ""),
|
||||
"release_created": bool(smart_release_result.get("release_created", False)),
|
||||
"release_deduplicated": bool(smart_release_result.get("release_deduplicated", False)),
|
||||
"rollout_created": bool(smart_release_result.get("rollout_created", False)),
|
||||
"requires_confirmation": bool(smart_release_result.get("requires_confirmation", False)),
|
||||
"blocked": bool(smart_release_result.get("blocked", False)),
|
||||
"rollout_id": int(rollout.get("id") or 0),
|
||||
"rollout_code": str(rollout.get("rollout_code") or ""),
|
||||
"smart_mode": str(smart_plan.get("mode") or ""),
|
||||
"smart_available": bool(smart_plan.get("available", False)),
|
||||
"target_node_codes": list((smart_plan.get("target_selector") or {}).get("node_codes") or []),
|
||||
"preview_warnings": list(preview.get("warnings") or []),
|
||||
"preview_approval_required": bool(preview.get("approval_required", False)),
|
||||
}
|
||||
|
||||
if ui_intent:
|
||||
summary["ui_intent"] = {
|
||||
"kind": str(ui_intent.get("kind") or ""),
|
||||
"release_id": int(ui_intent.get("release_id") or 0),
|
||||
"template_key": str(ui_intent.get("template_key") or ""),
|
||||
"playbook_key": str(ui_intent.get("playbook_key") or ""),
|
||||
"target_node_codes": list(ui_intent.get("target_node_codes") or []),
|
||||
}
|
||||
|
||||
if runtime_settings:
|
||||
summary["runtime_settings"] = {
|
||||
"worker_log_sync_enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
|
||||
"worker_log_sync_mode": str(runtime_settings.get("worker_log_sync_mode") or ""),
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
fi
|
||||
|
||||
if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
7776
domain-api/deploy/multi-region/drive_ops_center.sh
Executable file
7776
domain-api/deploy/multi-region/drive_ops_center.sh
Executable file
File diff suppressed because it is too large
Load Diff
476
domain-api/deploy/multi-region/drive_release_hub.sh
Executable file
476
domain-api/deploy/multi-region/drive_release_hub.sh
Executable file
@@ -0,0 +1,476 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib_ops_center.sh"
|
||||
ops_center_load_config "${SCRIPT_DIR}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
usage:
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh <base_url> <command> [args...]
|
||||
|
||||
commands:
|
||||
launchpad
|
||||
latest-package
|
||||
latest-release
|
||||
preview-latest
|
||||
preview-latest-smart <worker|control>
|
||||
preview-release <release_id> <worker|control>
|
||||
create-latest-release
|
||||
list-releases [limit]
|
||||
list-rollouts <release_id> [limit]
|
||||
smart-latest <worker|control> [confirm]
|
||||
smart-release <release_id> <worker|control> [confirm]
|
||||
|
||||
examples:
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 latest-package
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest-smart worker
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 create-latest-release
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-latest worker
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-release 18 control
|
||||
bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-release 18 control confirm
|
||||
EOF
|
||||
}
|
||||
|
||||
BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}"
|
||||
COMMAND="${2:-}"
|
||||
ARG1="${3:-}"
|
||||
ARG2="${4:-}"
|
||||
ARG3="${5:-}"
|
||||
REQUESTED_BY="${REQUESTED_BY:-$(ops_center_resolve_requested_by)}"
|
||||
|
||||
print_json_or_raw() {
|
||||
local raw_text="${1:-}"
|
||||
python3 - "${raw_text}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
raw_text = str(sys.argv[1] or "")
|
||||
try:
|
||||
payload = json.loads(raw_text)
|
||||
except Exception:
|
||||
print(raw_text)
|
||||
else:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
}
|
||||
|
||||
request_with_status() {
|
||||
local method="$1"
|
||||
local url="$2"
|
||||
local body="${3:-}"
|
||||
if [[ "${method}" == "POST" ]]; then
|
||||
curl -sS -X POST "${url}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "${body}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}'
|
||||
else
|
||||
curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}'
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -z "${COMMAND}" ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
METHOD="GET"
|
||||
PATH_SUFFIX=""
|
||||
REQUEST_JSON=""
|
||||
|
||||
confirm_to_bool() {
|
||||
local raw_value="${1:-}"
|
||||
case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in
|
||||
1|true|yes|y|confirm|confirmed)
|
||||
printf 'true'
|
||||
;;
|
||||
*)
|
||||
printf 'false'
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
case "${COMMAND}" in
|
||||
launchpad)
|
||||
CHANNEL="${ARG1:-stable}"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/launchpad?channel=${CHANNEL}"
|
||||
;;
|
||||
latest-package)
|
||||
PATH_SUFFIX="/api/v1/ops/releases/package-metadata/latest"
|
||||
;;
|
||||
latest-release)
|
||||
PATH_SUFFIX="/api/v1/ops/releases/latest"
|
||||
;;
|
||||
preview-latest)
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/preview"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub"
|
||||
payload = {
|
||||
"created_by": f"{requested_by}/latest-package-preview",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
preview-latest-smart)
|
||||
MODE="${ARG1:-}"
|
||||
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
|
||||
echo "mode must be worker or control" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout-preview"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
mode = str(sys.argv[1] or "").strip()
|
||||
requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub"
|
||||
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"created_by": f"{requested_by}/latest-package-preview",
|
||||
"rollout_created_by": f"{requested_by}/latest-package-preview/rollout",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
create-latest-release)
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub"
|
||||
payload = {
|
||||
"created_by": f"{requested_by}/latest-package",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
list-releases)
|
||||
RELEASE_LIMIT="${ARG1:-10}"
|
||||
PATH_SUFFIX="/api/v1/ops/releases?limit=${RELEASE_LIMIT}"
|
||||
;;
|
||||
list-rollouts)
|
||||
RELEASE_ID="${ARG1:-}"
|
||||
ROLLOUT_LIMIT="${ARG2:-10}"
|
||||
if [[ -z "${RELEASE_ID}" ]]; then
|
||||
echo "release_id is required for list-rollouts" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/rollouts?limit=${ROLLOUT_LIMIT}"
|
||||
;;
|
||||
smart-latest)
|
||||
MODE="${ARG1:-}"
|
||||
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
|
||||
echo "mode must be worker or control" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout"
|
||||
CONFIRM_RISKY="$(confirm_to_bool "${ARG2:-}")"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
mode = str(sys.argv[1] or "").strip()
|
||||
confirm_risky = str(sys.argv[2] or "").strip().lower() == "true"
|
||||
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub"
|
||||
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"confirm_risky": confirm_risky,
|
||||
"created_by": f"{requested_by}/latest-package",
|
||||
"rollout_created_by": f"{requested_by}/latest-package/rollout",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
preview-release)
|
||||
RELEASE_ID="${ARG1:-}"
|
||||
MODE="${ARG2:-}"
|
||||
if [[ -z "${RELEASE_ID}" ]]; then
|
||||
echo "release_id is required for preview-release" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
|
||||
echo "mode must be worker or control" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout-preview"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
mode = str(sys.argv[1] or "").strip()
|
||||
requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub"
|
||||
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"created_by": f"{requested_by}/existing-release-preview",
|
||||
"rollout_created_by": f"{requested_by}/existing-release-preview/rollout",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
smart-release)
|
||||
RELEASE_ID="${ARG1:-}"
|
||||
MODE="${ARG2:-}"
|
||||
if [[ -z "${RELEASE_ID}" ]]; then
|
||||
echo "release_id is required for smart-release" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then
|
||||
echo "mode must be worker or control" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
METHOD="POST"
|
||||
PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout"
|
||||
CONFIRM_RISKY="$(confirm_to_bool "${ARG3:-}")"
|
||||
REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}"
|
||||
import json
|
||||
import sys
|
||||
|
||||
mode = str(sys.argv[1] or "").strip()
|
||||
confirm_risky = str(sys.argv[2] or "").strip().lower() == "true"
|
||||
requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub"
|
||||
|
||||
payload = {
|
||||
"mode": mode,
|
||||
"confirm_risky": confirm_risky,
|
||||
"created_by": f"{requested_by}/existing-release",
|
||||
"rollout_created_by": f"{requested_by}/existing-release/rollout",
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
PY
|
||||
)"
|
||||
;;
|
||||
*)
|
||||
echo "unknown command: ${COMMAND}" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "[1/3] request"
|
||||
printf '%s\n' "method=${METHOD}"
|
||||
printf '%s\n' "url=${BASE_URL}${PATH_SUFFIX}"
|
||||
if [[ -n "${REQUEST_JSON}" ]]; then
|
||||
print_json_or_raw "${REQUEST_JSON}"
|
||||
else
|
||||
echo "(no request body)"
|
||||
fi
|
||||
echo
|
||||
echo
|
||||
|
||||
RAW_HTTP_RESPONSE="$(request_with_status "${METHOD}" "${BASE_URL}${PATH_SUFFIX}" "${REQUEST_JSON}")"
|
||||
HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}"
|
||||
RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}"
|
||||
|
||||
echo "[2/3] raw response"
|
||||
printf '%s\n' "http_status=${HTTP_STATUS}"
|
||||
print_json_or_raw "${RESPONSE_JSON}"
|
||||
echo
|
||||
echo
|
||||
|
||||
echo "[3/3] condensed summary"
|
||||
python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
raw_response = str(sys.argv[1] or "")
|
||||
http_status = int(sys.argv[2] or 0)
|
||||
try:
|
||||
payload = json.loads(raw_response)
|
||||
except Exception:
|
||||
print(json.dumps({
|
||||
"http_status": http_status,
|
||||
"code": 1,
|
||||
"message": "response is not valid json",
|
||||
"raw_response": raw_response[:500],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if 200 <= http_status < 300 else 1)
|
||||
|
||||
data = payload.get("data") or {}
|
||||
|
||||
raw_code = payload.get("code", 1)
|
||||
if raw_code in (0, "0", False):
|
||||
normalized_code = 0
|
||||
else:
|
||||
normalized_code = int(raw_code or 1)
|
||||
|
||||
summary = {
|
||||
"http_status": http_status,
|
||||
"code": normalized_code,
|
||||
"message": str(payload.get("message") or ""),
|
||||
}
|
||||
|
||||
if isinstance(data, dict) and "package" in data:
|
||||
package = dict(data.get("package") or {})
|
||||
summary["package"] = {
|
||||
"available": bool(package.get("available", False)),
|
||||
"package_name": str(package.get("package_name") or ""),
|
||||
"package_type": str(package.get("package_type") or ""),
|
||||
"generated_at": str(package.get("generated_at") or ""),
|
||||
"smoke_test_ok": bool(package.get("smoke_test_ok", False)),
|
||||
"smoke_test_message": str(package.get("smoke_test_message") or ""),
|
||||
}
|
||||
|
||||
if isinstance(data, dict) and "launchpad_status" in data:
|
||||
launchpad_status = dict(data.get("launchpad_status") or {})
|
||||
worker_preview = dict(data.get("worker_rollout_preview") or {})
|
||||
worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {})
|
||||
control_preview = dict(data.get("control_rollout_preview") or {})
|
||||
control_smart_rollout = dict(control_preview.get("smart_rollout") or {})
|
||||
summary["launchpad"] = {
|
||||
"channel": str(data.get("channel") or ""),
|
||||
"status": str(launchpad_status.get("status") or ""),
|
||||
"status_label": str(launchpad_status.get("status_label") or ""),
|
||||
"summary": str(launchpad_status.get("summary") or ""),
|
||||
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
|
||||
"recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""),
|
||||
"recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""),
|
||||
"recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""),
|
||||
"onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"recommended_command": str(launchpad_status.get("recommended_command") or ""),
|
||||
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
|
||||
"recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""),
|
||||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||||
"latest_package": dict(data.get("latest_package") or {}),
|
||||
"latest_release": {
|
||||
"id": int(((data.get("latest_release") or {}).get("id") or 0)),
|
||||
"release_version": str((data.get("latest_release") or {}).get("release_version") or ""),
|
||||
"status": str((data.get("latest_release") or {}).get("status") or ""),
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"message": str(worker_preview.get("message") or ""),
|
||||
"available": bool(worker_smart_rollout.get("available", False)),
|
||||
"requires_confirmation": bool(worker_preview.get("requires_confirmation", False)),
|
||||
"blocked": bool(worker_preview.get("blocked", False)),
|
||||
"execution_mode": str(worker_smart_rollout.get("execution_mode") or ""),
|
||||
"execution_mode_label": str(worker_smart_rollout.get("execution_mode_label") or ""),
|
||||
"target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []),
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"message": str(control_preview.get("message") or ""),
|
||||
"available": bool(control_smart_rollout.get("available", False)),
|
||||
"requires_confirmation": bool(control_preview.get("requires_confirmation", False)),
|
||||
"blocked": bool(control_preview.get("blocked", False)),
|
||||
"execution_mode": str(control_smart_rollout.get("execution_mode") or ""),
|
||||
"execution_mode_label": str(control_smart_rollout.get("execution_mode_label") or ""),
|
||||
"target_node_codes": list(control_smart_rollout.get("target_node_codes") or []),
|
||||
},
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
sys.exit(0)
|
||||
|
||||
if isinstance(data, dict) and "available" in data:
|
||||
summary["latest_package"] = {
|
||||
"available": bool(data.get("available", False)),
|
||||
"package_name": str(data.get("package_name") or ""),
|
||||
"package_type": str(data.get("package_type") or ""),
|
||||
"generated_at": str(data.get("generated_at") or ""),
|
||||
"commit_sha": str(data.get("commit_sha") or ""),
|
||||
"smoke_test_ok": bool(data.get("smoke_test_ok", False)),
|
||||
"smoke_test_message": str(data.get("smoke_test_message") or ""),
|
||||
"reason": str(data.get("reason") or ""),
|
||||
}
|
||||
elif isinstance(data, dict) and "releases" in data:
|
||||
releases = list(data.get("releases") or [])
|
||||
summary["releases"] = {
|
||||
"count": len(releases),
|
||||
"items": [
|
||||
{
|
||||
"id": int(item.get("id") or 0),
|
||||
"release_version": str(item.get("release_version") or ""),
|
||||
"channel": str(item.get("channel") or ""),
|
||||
"status": str(item.get("status") or ""),
|
||||
}
|
||||
for item in releases[:5]
|
||||
],
|
||||
}
|
||||
elif isinstance(data, dict) and "rollouts" in data:
|
||||
rollouts = list(data.get("rollouts") or [])
|
||||
summary["rollouts"] = {
|
||||
"count": len(rollouts),
|
||||
"items": [
|
||||
{
|
||||
"id": int(item.get("id") or 0),
|
||||
"rollout_code": str(item.get("rollout_code") or ""),
|
||||
"status": str(item.get("status") or ""),
|
||||
"jobs_total": int(item.get("jobs_total") or 0),
|
||||
"jobs_created": int(item.get("jobs_created") or 0),
|
||||
}
|
||||
for item in rollouts[:5]
|
||||
],
|
||||
}
|
||||
else:
|
||||
release = data.get("release") or data
|
||||
rollout = data.get("rollout") or {}
|
||||
smart_rollout = data.get("smart_rollout") or {}
|
||||
preview = data.get("preview") or {}
|
||||
summary["preview_source"] = str(data.get("preview_source") or "")
|
||||
summary["release"] = {
|
||||
"id": int(release.get("id") or 0),
|
||||
"release_version": str(release.get("release_version") or ""),
|
||||
"channel": str(release.get("channel") or ""),
|
||||
"status": str(release.get("status") or ""),
|
||||
"is_preview": bool(release.get("is_preview", False)),
|
||||
}
|
||||
if rollout:
|
||||
summary["rollout"] = {
|
||||
"id": int(rollout.get("id") or 0),
|
||||
"rollout_code": str(rollout.get("rollout_code") or ""),
|
||||
"status": str(rollout.get("status") or ""),
|
||||
}
|
||||
if smart_rollout:
|
||||
summary["smart_rollout"] = {
|
||||
"available": bool(smart_rollout.get("available", False)),
|
||||
"mode": str(smart_rollout.get("mode") or ""),
|
||||
"mode_label": str(smart_rollout.get("mode_label") or ""),
|
||||
"target_node_codes": list((smart_rollout.get("target_selector") or {}).get("node_codes") or []),
|
||||
"summary": str(smart_rollout.get("summary") or smart_rollout.get("reason") or ""),
|
||||
}
|
||||
if preview:
|
||||
summary["preview"] = {
|
||||
"blocked": bool(preview.get("blocked", False)),
|
||||
"approval_required": bool(preview.get("approval_required", False)),
|
||||
"warnings": list(preview.get("warnings") or []),
|
||||
}
|
||||
summary["result_flags"] = {
|
||||
"release_created": bool(data.get("release_created", False)),
|
||||
"release_deduplicated": bool(data.get("release_deduplicated", False)),
|
||||
"rollout_created": bool(data.get("rollout_created", False)),
|
||||
"requires_confirmation": bool(data.get("requires_confirmation", False)),
|
||||
"blocked": bool(data.get("blocked", False)),
|
||||
}
|
||||
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
|
||||
if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
556
domain-api/deploy/multi-region/export_go_live_bundle.sh
Executable file
556
domain-api/deploy/multi-region/export_go_live_bundle.sh
Executable file
@@ -0,0 +1,556 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/lib_ops_center.sh"
|
||||
ops_center_load_config "${SCRIPT_DIR}"
|
||||
|
||||
MAINLAND_BASE_URL_DEFAULT="$(ops_center_resolve_mainland_api_base_url)"
|
||||
OVERSEAS_BASE_URL_DEFAULT="$(ops_center_resolve_overseas_api_base_url)"
|
||||
REPORT_ROOT_DEFAULT="${OPS_CENTER_REPORT_ROOT}/go-live-bundles"
|
||||
|
||||
REPORT_DIR_INPUT="${1:-}"
|
||||
MAINLAND_BASE_URL_INPUT="${2:-}"
|
||||
OVERSEAS_BASE_URL_INPUT="${3:-}"
|
||||
|
||||
timestamp_now() {
|
||||
date '+%Y%m%d_%H%M%S'
|
||||
}
|
||||
|
||||
normalize_optional_base_url() {
|
||||
local raw_value="${1:-}"
|
||||
local fallback_value="${2:-}"
|
||||
if [[ -z "${raw_value}" ]]; then
|
||||
printf '%s' "${fallback_value}"
|
||||
return
|
||||
fi
|
||||
case "${raw_value}" in
|
||||
-|none|null|disabled)
|
||||
printf ''
|
||||
;;
|
||||
*)
|
||||
printf '%s' "${raw_value}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
if [[ -n "${REPORT_DIR_INPUT}" && "${REPORT_DIR_INPUT}" != http://* && "${REPORT_DIR_INPUT}" != https://* ]]; then
|
||||
REPORT_DIR="${REPORT_DIR_INPUT}"
|
||||
MAINLAND_BASE_URL="${MAINLAND_BASE_URL_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}"
|
||||
OVERSEAS_BASE_URL="$(normalize_optional_base_url "${OVERSEAS_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")"
|
||||
else
|
||||
REPORT_DIR="${REPORT_ROOT_DEFAULT}/go-live-bundle-$(timestamp_now)"
|
||||
MAINLAND_BASE_URL="${REPORT_DIR_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}"
|
||||
OVERSEAS_BASE_URL="$(normalize_optional_base_url "${MAINLAND_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")"
|
||||
fi
|
||||
|
||||
mkdir -p "${REPORT_DIR}"
|
||||
|
||||
CURL_CONNECT_TIMEOUT="${OPS_CENTER_CURL_CONNECT_TIMEOUT:-3}"
|
||||
CURL_MAX_TIME="${OPS_CENTER_CURL_MAX_TIME:-15}"
|
||||
STEP_COMMAND_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS:-45}"
|
||||
DOCTOR_DECISION_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_DECISION_TIMEOUT_SECONDS:-120}"
|
||||
DOCTOR_EXPORT_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_EXPORT_TIMEOUT_SECONDS:-180}"
|
||||
DOCTOR_INNER_REPORT_TIMEOUT_SECONDS="${OPS_CENTER_DOCTOR_INNER_REPORT_TIMEOUT_SECONDS:-60}"
|
||||
|
||||
ENV_AUDIT_TXT_PATH="${REPORT_DIR}/00_env_audit.txt"
|
||||
SUMMARY_TXT_PATH="${REPORT_DIR}/01_go_live_check_summary.txt"
|
||||
GO_LIVE_JSON_PATH="${REPORT_DIR}/02_go_live_summary.json"
|
||||
STACK_JSON_PATH="${REPORT_DIR}/03_stack_diagnosis.json"
|
||||
DRIVER_FEED_JSON_PATH="${REPORT_DIR}/04_driver_feed.json"
|
||||
CODEX_BRIEF_JSON_PATH="${REPORT_DIR}/05_codex_brief.json"
|
||||
RELEASE_LAUNCHPAD_JSON_PATH="${REPORT_DIR}/06_release_launchpad.json"
|
||||
DOCTOR_DECISION_JSON_PATH="${REPORT_DIR}/07_doctor_decision.json"
|
||||
DOCTOR_EXPORT_STDOUT_PATH="${REPORT_DIR}/08_doctor_export_stdout.txt"
|
||||
MANIFEST_PATH="${REPORT_DIR}/manifest.json"
|
||||
|
||||
declare -a STEP_RESULTS=()
|
||||
|
||||
pretty_print_json_file() {
|
||||
local source_path="${1:-}"
|
||||
python3 - "${source_path}" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
raw_text = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
try:
|
||||
payload = json.loads(raw_text)
|
||||
except Exception:
|
||||
print(raw_text)
|
||||
else:
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
}
|
||||
|
||||
record_step_result() {
|
||||
local key="${1:-}"
|
||||
local path="${2:-}"
|
||||
local ok="${3:-false}"
|
||||
local http_status="${4:-}"
|
||||
local exit_code="${5:-0}"
|
||||
STEP_RESULTS+=("${key}|${path}|${ok}|${http_status}|${exit_code}")
|
||||
}
|
||||
|
||||
fetch_json_to_file() {
|
||||
local key="${1:-}"
|
||||
local url="${2:-}"
|
||||
local output_path="${3:-}"
|
||||
local tmp_path
|
||||
tmp_path="$(mktemp)"
|
||||
local raw_text http_status
|
||||
raw_text="$(curl -sS \
|
||||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||||
--max-time "${CURL_MAX_TIME}" \
|
||||
"${url}" \
|
||||
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
||||
http_status="${raw_text##*$'\n'HTTP_STATUS=}"
|
||||
printf '%s' "${raw_text%$'\n'HTTP_STATUS=*}" > "${tmp_path}"
|
||||
pretty_print_json_file "${tmp_path}" > "${output_path}"
|
||||
rm -f "${tmp_path}"
|
||||
if [[ "${http_status}" == 2* ]]; then
|
||||
record_step_result "${key}" "${output_path}" "true" "${http_status}" "0"
|
||||
return 0
|
||||
fi
|
||||
record_step_result "${key}" "${output_path}" "false" "${http_status}" "1"
|
||||
return 1
|
||||
}
|
||||
|
||||
capture_command_to_file() {
|
||||
local key="${1:-}"
|
||||
local output_path="${2:-}"
|
||||
shift 2 || true
|
||||
capture_command_to_file_with_timeout "${STEP_COMMAND_TIMEOUT_SECONDS}" "${key}" "${output_path}" "$@"
|
||||
}
|
||||
|
||||
capture_command_to_file_with_timeout() {
|
||||
local timeout_seconds="${1:-45}"
|
||||
local key="${2:-}"
|
||||
local output_path="${3:-}"
|
||||
shift 3 || true
|
||||
local -a wrapped_command=("$@")
|
||||
local had_errexit="false"
|
||||
case "$-" in
|
||||
*e*) had_errexit="true" ;;
|
||||
esac
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
wrapped_command=(timeout "${timeout_seconds}" "$@")
|
||||
fi
|
||||
set +e
|
||||
"${wrapped_command[@]}" > "${output_path}" 2>&1
|
||||
local exit_code=$?
|
||||
if [[ "${had_errexit}" == "true" ]]; then
|
||||
set -e
|
||||
else
|
||||
set +e
|
||||
fi
|
||||
if [[ "${exit_code}" -eq 0 ]]; then
|
||||
record_step_result "${key}" "${output_path}" "true" "" "${exit_code}"
|
||||
return 0
|
||||
fi
|
||||
record_step_result "${key}" "${output_path}" "false" "" "${exit_code}"
|
||||
return "${exit_code}"
|
||||
}
|
||||
|
||||
materialize_go_live_summary_fallback() {
|
||||
local summary_path="${1:-}"
|
||||
local output_path="${2:-}"
|
||||
python3 - "${summary_path}" "${output_path}" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
summary_path = pathlib.Path(sys.argv[1])
|
||||
output_path = pathlib.Path(sys.argv[2])
|
||||
raw_text = summary_path.read_text(encoding="utf-8") if summary_path.exists() else ""
|
||||
start = raw_text.find("{")
|
||||
end = raw_text.rfind("}")
|
||||
if start < 0 or end < start:
|
||||
sys.exit(1)
|
||||
json_text = raw_text[start:end + 1]
|
||||
payload = json.loads(json_text)
|
||||
if not isinstance(payload, dict):
|
||||
sys.exit(1)
|
||||
output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
PY
|
||||
}
|
||||
|
||||
override_step_result() {
|
||||
local key="${1:-}"
|
||||
local path="${2:-}"
|
||||
local ok="${3:-false}"
|
||||
local http_status="${4:-}"
|
||||
local exit_code="${5:-0}"
|
||||
local -a updated_results=()
|
||||
local raw_item existing_key
|
||||
for raw_item in "${STEP_RESULTS[@]}"; do
|
||||
existing_key="${raw_item%%|*}"
|
||||
if [[ "${existing_key}" == "${key}" ]]; then
|
||||
continue
|
||||
fi
|
||||
updated_results+=("${raw_item}")
|
||||
done
|
||||
STEP_RESULTS=("${updated_results[@]}")
|
||||
record_step_result "${key}" "${path}" "${ok}" "${http_status}" "${exit_code}"
|
||||
}
|
||||
|
||||
set +e
|
||||
capture_command_to_file \
|
||||
"env_audit" \
|
||||
"${ENV_AUDIT_TXT_PATH}" \
|
||||
bash "${SCRIPT_DIR}/drive_ops_center.sh" env-audit "${MAINLAND_BASE_URL}"
|
||||
|
||||
capture_command_to_file \
|
||||
"go_live_check_summary" \
|
||||
"${SUMMARY_TXT_PATH}" \
|
||||
bash "${SCRIPT_DIR}/drive_ops_center.sh" go-live-check "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" summary
|
||||
|
||||
fetch_json_to_file \
|
||||
"go_live_summary" \
|
||||
"${MAINLAND_BASE_URL}/api/v1/ops/go-live-summary?base_url=${MAINLAND_BASE_URL}" \
|
||||
"${GO_LIVE_JSON_PATH}"
|
||||
|
||||
if [[ ! -s "${GO_LIVE_JSON_PATH}" || "$(sed -n '1p' "${GO_LIVE_JSON_PATH}" 2>/dev/null)" != "{"* ]]; then
|
||||
if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then
|
||||
override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$(python3 - "${GO_LIVE_JSON_PATH}" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
path = pathlib.Path(sys.argv[1])
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
print("invalid")
|
||||
raise SystemExit(0)
|
||||
|
||||
if isinstance(payload, dict) and payload.get("detail") == "Not Found":
|
||||
print("not_found")
|
||||
else:
|
||||
print("ok")
|
||||
PY
|
||||
)" == "not_found" ]]; then
|
||||
if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then
|
||||
override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0"
|
||||
fi
|
||||
fi
|
||||
|
||||
fetch_json_to_file \
|
||||
"stack_diagnosis" \
|
||||
"${MAINLAND_BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${MAINLAND_BASE_URL}" \
|
||||
"${STACK_JSON_PATH}"
|
||||
|
||||
fetch_json_to_file \
|
||||
"driver_feed" \
|
||||
"${MAINLAND_BASE_URL}/api/v1/ops/driver-feed" \
|
||||
"${DRIVER_FEED_JSON_PATH}"
|
||||
|
||||
fetch_json_to_file \
|
||||
"codex_brief" \
|
||||
"${MAINLAND_BASE_URL}/api/v1/ops/codex-brief" \
|
||||
"${CODEX_BRIEF_JSON_PATH}"
|
||||
|
||||
fetch_json_to_file \
|
||||
"release_launchpad" \
|
||||
"${MAINLAND_BASE_URL}/api/v1/ops/releases/launchpad" \
|
||||
"${RELEASE_LAUNCHPAD_JSON_PATH}"
|
||||
|
||||
capture_command_to_file_with_timeout \
|
||||
"${DOCTOR_EXPORT_TIMEOUT_SECONDS}" \
|
||||
"doctor_export" \
|
||||
"${DOCTOR_EXPORT_STDOUT_PATH}" \
|
||||
env "OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS=${DOCTOR_INNER_REPORT_TIMEOUT_SECONDS}" \
|
||||
bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-export "${REPORT_DIR}/doctor-export" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}"
|
||||
|
||||
capture_command_to_file_with_timeout \
|
||||
"${DOCTOR_DECISION_TIMEOUT_SECONDS}" \
|
||||
"doctor_decision" \
|
||||
"${DOCTOR_DECISION_JSON_PATH}" \
|
||||
bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${REPORT_DIR}/doctor-export"
|
||||
set -e
|
||||
|
||||
python3 - <<'PY' \
|
||||
"${MANIFEST_PATH}" \
|
||||
"${REPORT_DIR}" \
|
||||
"${MAINLAND_BASE_URL}" \
|
||||
"${OVERSEAS_BASE_URL}" \
|
||||
"${ENV_AUDIT_TXT_PATH}" \
|
||||
"${SUMMARY_TXT_PATH}" \
|
||||
"${GO_LIVE_JSON_PATH}" \
|
||||
"${STACK_JSON_PATH}" \
|
||||
"${DRIVER_FEED_JSON_PATH}" \
|
||||
"${CODEX_BRIEF_JSON_PATH}" \
|
||||
"${RELEASE_LAUNCHPAD_JSON_PATH}" \
|
||||
"${DOCTOR_DECISION_JSON_PATH}" \
|
||||
"${DOCTOR_EXPORT_STDOUT_PATH}" \
|
||||
"${STEP_RESULTS[@]}"
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
manifest_path = pathlib.Path(sys.argv[1])
|
||||
report_dir = pathlib.Path(sys.argv[2])
|
||||
mainland_base_url = sys.argv[3]
|
||||
overseas_base_url = sys.argv[4]
|
||||
|
||||
file_keys = [
|
||||
"env_audit",
|
||||
"go_live_check_summary",
|
||||
"go_live_summary",
|
||||
"stack_diagnosis",
|
||||
"driver_feed",
|
||||
"codex_brief",
|
||||
"release_launchpad",
|
||||
"doctor_decision",
|
||||
"doctor_export",
|
||||
]
|
||||
file_paths = {
|
||||
"env_audit": pathlib.Path(sys.argv[5]),
|
||||
"go_live_check_summary": pathlib.Path(sys.argv[6]),
|
||||
"go_live_summary": pathlib.Path(sys.argv[7]),
|
||||
"stack_diagnosis": pathlib.Path(sys.argv[8]),
|
||||
"driver_feed": pathlib.Path(sys.argv[9]),
|
||||
"codex_brief": pathlib.Path(sys.argv[10]),
|
||||
"release_launchpad": pathlib.Path(sys.argv[11]),
|
||||
"doctor_decision": pathlib.Path(sys.argv[12]),
|
||||
"doctor_export": pathlib.Path(sys.argv[13]),
|
||||
}
|
||||
|
||||
step_results = {}
|
||||
for raw_item in sys.argv[14:]:
|
||||
parts = raw_item.split("|", 4)
|
||||
if len(parts) != 5:
|
||||
continue
|
||||
key, path_value, ok_value, http_status, exit_code = parts
|
||||
step_results[key] = {
|
||||
"path": path_value,
|
||||
"ok": ok_value == "true",
|
||||
"http_status": http_status,
|
||||
"exit_code": int(exit_code or 0),
|
||||
}
|
||||
|
||||
|
||||
def load_json_file(path: pathlib.Path) -> dict:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_env_audit_summary(path: pathlib.Path) -> dict:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
raw_text = path.read_text(encoding="utf-8", errors="replace")
|
||||
marker = "[8/8] condensed env audit summary"
|
||||
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
|
||||
if not candidate_text:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(candidate_text)
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
failures = []
|
||||
artifacts = []
|
||||
for key in file_keys:
|
||||
artifact_path = file_paths[key]
|
||||
result = dict(step_results.get(key) or {})
|
||||
if not result:
|
||||
result = {
|
||||
"path": str(artifact_path),
|
||||
"ok": artifact_path.exists(),
|
||||
"http_status": "",
|
||||
"exit_code": 0 if artifact_path.exists() else 1,
|
||||
}
|
||||
entry = {
|
||||
"key": key,
|
||||
"path": str(artifact_path),
|
||||
"exists": artifact_path.exists(),
|
||||
"ok": bool(result.get("ok", False)),
|
||||
"http_status": str(result.get("http_status") or ""),
|
||||
"exit_code": int(result.get("exit_code", 0) or 0),
|
||||
}
|
||||
artifacts.append(entry)
|
||||
if not entry["ok"]:
|
||||
failures.append(key)
|
||||
|
||||
go_live_summary_payload = load_json_file(file_paths["go_live_summary"])
|
||||
stack_diagnosis_payload = load_json_file(file_paths["stack_diagnosis"])
|
||||
driver_feed_payload = load_json_file(file_paths["driver_feed"])
|
||||
codex_brief_payload = load_json_file(file_paths["codex_brief"])
|
||||
env_audit_summary = load_env_audit_summary(file_paths["env_audit"])
|
||||
|
||||
stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {})
|
||||
driver_feed_summary = dict(driver_feed_payload.get("summary") or {})
|
||||
codex_brief_summary = dict(codex_brief_payload.get("summary") or {})
|
||||
|
||||
|
||||
def first_nonempty(*values: object) -> str:
|
||||
for value in values:
|
||||
normalized = str(value or "").strip()
|
||||
if normalized:
|
||||
return normalized
|
||||
return ""
|
||||
|
||||
|
||||
def nonempty_distinct(values: dict[str, str]) -> list[str]:
|
||||
return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()})
|
||||
|
||||
|
||||
def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]:
|
||||
return sorted({int(values[source]) for source in values if source in available_sources})
|
||||
|
||||
|
||||
launchpad_target_node_codes = {
|
||||
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
}
|
||||
launchpad_recovery_labels = {
|
||||
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
}
|
||||
launchpad_recovery_summaries = {
|
||||
"go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
}
|
||||
launchpad_bootstrap_pending_nodes = {
|
||||
"go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
}
|
||||
launchpad_acceptance_ready_nodes = {
|
||||
"go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
}
|
||||
available_alignment_sources = {
|
||||
source
|
||||
for source, payload_value in {
|
||||
"go_live_summary": go_live_summary_payload,
|
||||
"stack_diagnosis": stack_diagnosis_payload,
|
||||
"driver_feed": driver_feed_payload,
|
||||
"codex_brief": codex_brief_payload,
|
||||
}.items()
|
||||
if isinstance(payload_value, dict) and payload_value
|
||||
}
|
||||
launchpad_alignment = {
|
||||
"consistent": True,
|
||||
"target_node_code_consistent": True,
|
||||
"recovery_label_consistent": True,
|
||||
"recovery_summary_consistent": True,
|
||||
"bootstrap_pending_consistent": True,
|
||||
"acceptance_ready_consistent": True,
|
||||
"available_sources": sorted(available_alignment_sources),
|
||||
"target_node_codes": launchpad_target_node_codes,
|
||||
"recovery_labels": launchpad_recovery_labels,
|
||||
"recovery_summaries": launchpad_recovery_summaries,
|
||||
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
|
||||
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
|
||||
}
|
||||
launchpad_alignment["target_node_code_consistent"] = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_alignment["recovery_label_consistent"] = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_alignment["recovery_summary_consistent"] = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_alignment["bootstrap_pending_consistent"] = len(
|
||||
distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources)
|
||||
) <= 1
|
||||
launchpad_alignment["acceptance_ready_consistent"] = len(
|
||||
distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources)
|
||||
) <= 1
|
||||
launchpad_alignment["consistent"] = all([
|
||||
bool(launchpad_alignment["target_node_code_consistent"]),
|
||||
bool(launchpad_alignment["recovery_label_consistent"]),
|
||||
bool(launchpad_alignment["recovery_summary_consistent"]),
|
||||
bool(launchpad_alignment["bootstrap_pending_consistent"]),
|
||||
bool(launchpad_alignment["acceptance_ready_consistent"]),
|
||||
])
|
||||
|
||||
env_audit_status = str(env_audit_summary.get("status") or "").strip()
|
||||
env_audit_runtime_may_need_restart = bool(((env_audit_summary.get("runtime") or {}).get("runtime_may_need_restart", False)))
|
||||
review_status_hint = "ready"
|
||||
review_headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
|
||||
if failures or env_audit_status == "blocked":
|
||||
review_status_hint = "blocked"
|
||||
review_headline = "当前导出包仍有关键阻断项,不能直接作为正式上线终稿。"
|
||||
elif env_audit_status == "attention" or not launchpad_alignment["consistent"]:
|
||||
review_status_hint = "attention"
|
||||
if not launchpad_alignment["consistent"]:
|
||||
review_headline = "核心报告虽已导出,但 launchpad 摘要在不同 surface 之间存在漂移,建议先重新复核。"
|
||||
else:
|
||||
review_headline = "核心报告已齐,但环境审计仍提示存在待收口事项。"
|
||||
|
||||
payload = {
|
||||
"report_dir": str(report_dir),
|
||||
"mainland_base_url": mainland_base_url,
|
||||
"overseas_base_url": overseas_base_url,
|
||||
"artifacts": artifacts,
|
||||
"failures": failures,
|
||||
"summary": {
|
||||
"ok": len(failures) == 0,
|
||||
"artifact_total": len(artifacts),
|
||||
"failure_total": len(failures),
|
||||
"review_status_hint": review_status_hint,
|
||||
"review_headline": review_headline,
|
||||
"env_audit_status": env_audit_status,
|
||||
"env_audit_runtime_may_need_restart": env_audit_runtime_may_need_restart,
|
||||
"launchpad_recommended_target_node_code": first_nonempty(
|
||||
go_live_summary_payload.get("launchpad_recommended_target_node_code"),
|
||||
stack_diagnosis.get("launchpad_recommended_target_node_code"),
|
||||
driver_feed_summary.get("launchpad_recommended_target_node_code"),
|
||||
codex_brief_summary.get("launchpad_recommended_target_node_code"),
|
||||
),
|
||||
"launchpad_recommended_recovery_label": first_nonempty(
|
||||
go_live_summary_payload.get("launchpad_recommended_recovery_label"),
|
||||
stack_diagnosis.get("launchpad_recommended_recovery_label"),
|
||||
driver_feed_summary.get("launchpad_recommended_recovery_label"),
|
||||
codex_brief_summary.get("launchpad_recommended_recovery_label"),
|
||||
),
|
||||
"launchpad_recommended_recovery_summary": first_nonempty(
|
||||
go_live_summary_payload.get("launchpad_recommended_recovery_summary"),
|
||||
stack_diagnosis.get("launchpad_recommended_recovery_summary"),
|
||||
driver_feed_summary.get("launchpad_recommended_recovery_summary"),
|
||||
codex_brief_summary.get("launchpad_recommended_recovery_summary"),
|
||||
),
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": max(launchpad_bootstrap_pending_nodes.values() or [0]),
|
||||
"launchpad_onboarding_acceptance_ready_nodes": max(launchpad_acceptance_ready_nodes.values() or [0]),
|
||||
"launchpad_alignment": launchpad_alignment,
|
||||
"recommended_reading_order": [
|
||||
"00_env_audit.txt",
|
||||
"01_go_live_check_summary.txt",
|
||||
"02_go_live_summary.json",
|
||||
"03_stack_diagnosis.json",
|
||||
"04_driver_feed.json",
|
||||
"05_codex_brief.json",
|
||||
"06_release_launchpad.json",
|
||||
"07_doctor_decision.json",
|
||||
"08_doctor_export_stdout.txt",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
135
domain-api/deploy/multi-region/init_ops_center_config.sh
Executable file
135
domain-api/deploy/multi-region/init_ops_center_config.sh
Executable file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEMPLATE_PATH="${SCRIPT_DIR}/templates/domaincheck-ops-center.env.example"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
usage:
|
||||
bash domain-api/deploy/multi-region/init_ops_center_config.sh \
|
||||
[config_path] [mainland_api_base_url] [overseas_api_base_url] [control_plane_base_url]
|
||||
|
||||
examples:
|
||||
bash domain-api/deploy/multi-region/init_ops_center_config.sh
|
||||
bash domain-api/deploy/multi-region/init_ops_center_config.sh /etc/default/domaincheck-ops-center \
|
||||
http://121.204.244.188:8100 \
|
||||
http://152.53.37.118:8100 \
|
||||
https://api.example.com
|
||||
|
||||
notes:
|
||||
- existing config files are preserved unless FORCE=true
|
||||
- advanced fields can be overridden with env vars:
|
||||
OPS_ACTIVE_PROFILE
|
||||
OPS_PROFILE_NAMES
|
||||
OPS_DEFAULT_REQUESTED_BY
|
||||
OPS_DEFAULT_ISSUED_BY
|
||||
OPS_DEFAULT_EXPIRES_IN_HOURS
|
||||
OPS_DEFAULT_ROOT_DIR
|
||||
OPS_REPORT_ROOT
|
||||
EOF
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TARGET_PATH="${1:-/etc/default/domaincheck-ops-center}"
|
||||
MAINLAND_API_BASE_URL="${2:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}"
|
||||
OVERSEAS_API_BASE_URL="${3:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}"
|
||||
CONTROL_PLANE_BASE_URL="${4:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}"
|
||||
|
||||
ACTIVE_PROFILE="${OPS_ACTIVE_PROFILE:-default}"
|
||||
PROFILE_NAMES="${OPS_PROFILE_NAMES:-${ACTIVE_PROFILE}}"
|
||||
DEFAULT_REQUESTED_BY="${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}"
|
||||
DEFAULT_ISSUED_BY="${OPS_DEFAULT_ISSUED_BY:-ops-center}"
|
||||
DEFAULT_EXPIRES_IN_HOURS="${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}"
|
||||
DEFAULT_ROOT_DIR="${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}"
|
||||
REPORT_ROOT="${OPS_REPORT_ROOT:-}"
|
||||
NODE_AGENT_SERVICE_NAME="${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}"
|
||||
NODE_AGENT_ENV_FILE="${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}"
|
||||
FORCE_WRITE="${FORCE:-false}"
|
||||
|
||||
if [[ ! -f "${TEMPLATE_PATH}" ]]; then
|
||||
echo "template not found: ${TEMPLATE_PATH}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "${TARGET_PATH}")"
|
||||
|
||||
if [[ ! -f "${TARGET_PATH}" || "${FORCE_WRITE}" == "true" ]]; then
|
||||
cp "${TEMPLATE_PATH}" "${TARGET_PATH}"
|
||||
fi
|
||||
|
||||
upsert_key() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
if grep -q "^${key}=" "${TARGET_PATH}"; then
|
||||
sed -i "s#^${key}=.*#${key}=${value}#" "${TARGET_PATH}"
|
||||
else
|
||||
printf '%s=%s\n' "${key}" "${value}" >>"${TARGET_PATH}"
|
||||
fi
|
||||
}
|
||||
|
||||
upsert_commented_key() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
if [[ -z "${value}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
upsert_key "${key}" "${value}"
|
||||
}
|
||||
|
||||
upsert_key "OPS_ACTIVE_PROFILE" "${ACTIVE_PROFILE}"
|
||||
upsert_key "OPS_PROFILE_NAMES" "${PROFILE_NAMES}"
|
||||
upsert_key "OPS_MAINLAND_API_BASE_URL" "${MAINLAND_API_BASE_URL}"
|
||||
upsert_key "OPS_OVERSEAS_API_BASE_URL" "${OVERSEAS_API_BASE_URL}"
|
||||
upsert_key "OPS_DEFAULT_CONTROL_PLANE_BASE_URL" "${CONTROL_PLANE_BASE_URL}"
|
||||
upsert_key "OPS_DEFAULT_REQUESTED_BY" "${DEFAULT_REQUESTED_BY}"
|
||||
upsert_key "OPS_DEFAULT_ISSUED_BY" "${DEFAULT_ISSUED_BY}"
|
||||
upsert_key "OPS_DEFAULT_EXPIRES_IN_HOURS" "${DEFAULT_EXPIRES_IN_HOURS}"
|
||||
upsert_key "OPS_DEFAULT_ROOT_DIR" "${DEFAULT_ROOT_DIR}"
|
||||
upsert_key "OPS_NODE_AGENT_SERVICE_NAME" "${NODE_AGENT_SERVICE_NAME}"
|
||||
upsert_key "OPS_NODE_AGENT_ENV_FILE" "${NODE_AGENT_ENV_FILE}"
|
||||
upsert_commented_key "OPS_REPORT_ROOT" "${REPORT_ROOT}"
|
||||
|
||||
python3 - <<'PY' "${TARGET_PATH}"
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
config_path = Path(sys.argv[1])
|
||||
keys = [
|
||||
"OPS_ACTIVE_PROFILE",
|
||||
"OPS_PROFILE_NAMES",
|
||||
"OPS_MAINLAND_API_BASE_URL",
|
||||
"OPS_OVERSEAS_API_BASE_URL",
|
||||
"OPS_DEFAULT_CONTROL_PLANE_BASE_URL",
|
||||
"OPS_DEFAULT_REQUESTED_BY",
|
||||
"OPS_DEFAULT_ISSUED_BY",
|
||||
"OPS_DEFAULT_EXPIRES_IN_HOURS",
|
||||
"OPS_DEFAULT_ROOT_DIR",
|
||||
"OPS_NODE_AGENT_SERVICE_NAME",
|
||||
"OPS_NODE_AGENT_ENV_FILE",
|
||||
"OPS_REPORT_ROOT",
|
||||
]
|
||||
values = {}
|
||||
for line in config_path.read_text(encoding="utf-8").splitlines():
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
if key in keys:
|
||||
values[key] = value
|
||||
|
||||
print(json.dumps({
|
||||
"ok": True,
|
||||
"config_path": str(config_path),
|
||||
"summary": values,
|
||||
"next_commands": [
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh config",
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export",
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh agent-check",
|
||||
],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
@@ -100,7 +100,7 @@ REDIS_DB=0
|
||||
EOF
|
||||
|
||||
cd "${DOMAIN_API_DIR}"
|
||||
bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller
|
||||
bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller
|
||||
|
||||
cat >/etc/default/domaincheck-worker <<EOF
|
||||
WORKER_MODE=linux-systemd
|
||||
@@ -138,4 +138,4 @@ chown -R www:www "${REPO_DIR}"
|
||||
|
||||
echo
|
||||
echo "国内 controller 部署完成,建议继续执行:"
|
||||
echo "cd ${DOMAIN_API_DIR} && bash deploy/multi-region/check_mainland_controller.sh"
|
||||
echo "cd ${DOMAIN_API_DIR} && bash domain-api/deploy/multi-region/check_mainland_controller.sh"
|
||||
|
||||
@@ -101,7 +101,7 @@ REDIS_DB=0
|
||||
EOF
|
||||
|
||||
cd "${DOMAIN_API_DIR}"
|
||||
bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker
|
||||
bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker
|
||||
|
||||
cat >/etc/default/domaincheck-worker <<EOF
|
||||
WORKER_MODE=linux-systemd
|
||||
|
||||
@@ -185,6 +185,15 @@ sed -i 's#^NODE_ROLE=.*#NODE_ROLE=control#' /etc/default/domaincheck-api
|
||||
sed -i 's#^SYNC_PUSH_ENABLED=.*#SYNC_PUSH_ENABLED=false#' /etc/default/domaincheck-api
|
||||
sed -i "s#^CORS_ORIGINS=.*#CORS_ORIGINS='[\\\"https://${ADMIN_DOMAIN}\\\",\\\"http://127.0.0.1:3201\\\",\\\"http://localhost:3201\\\"]'#" /etc/default/domaincheck-api
|
||||
|
||||
OPS_DEFAULT_REQUESTED_BY="cli/${OVERSEAS_NODE_CODE}" \
|
||||
OPS_DEFAULT_ISSUED_BY="${OVERSEAS_NODE_CODE}" \
|
||||
OPS_DEFAULT_ROOT_DIR="${BASE_DIR}" \
|
||||
bash "${DOMAIN_API_DIR}/deploy/multi-region/init_ops_center_config.sh" \
|
||||
/etc/default/domaincheck-ops-center \
|
||||
http://127.0.0.1:8100 \
|
||||
http://127.0.0.1:8100 \
|
||||
"https://${API_DOMAIN}" >/dev/null
|
||||
|
||||
cp "${DOMAIN_API_DIR}/deploy/systemd/domain-api.service" /etc/systemd/system/domaincheck-api.service
|
||||
cp "${DOMAIN_API_DIR}/deploy/systemd/domain-worker.service" /etc/systemd/system/domaincheck-worker.service
|
||||
|
||||
@@ -206,3 +215,5 @@ echo "国外主控机部署完成,建议继续执行:"
|
||||
echo "curl http://127.0.0.1:8100/health"
|
||||
echo "curl http://127.0.0.1:8100/api/v1/runtime/preflight"
|
||||
echo "curl http://127.0.0.1:8100/api/v1/runtime/readiness"
|
||||
echo "bash ${DOMAIN_API_DIR}/deploy/multi-region/drive_ops_center.sh config"
|
||||
echo "如后续接入大陆 controller,请记得修改 /etc/default/domaincheck-ops-center 中的 OPS_MAINLAND_API_BASE_URL"
|
||||
|
||||
106
domain-api/deploy/multi-region/lib_ops_center.sh
Normal file
106
domain-api/deploy/multi-region/lib_ops_center.sh
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
ops_center_normalize_profile_key() {
|
||||
local raw_value="${1:-default}"
|
||||
printf '%s' "${raw_value}" | tr '[:lower:]-./' '[:upper:]____' | tr -cd 'A-Z0-9_'
|
||||
}
|
||||
|
||||
ops_center_load_config() {
|
||||
local caller_dir="${1:-}"
|
||||
if [[ -z "${caller_dir}" ]]; then
|
||||
caller_dir="$(pwd)"
|
||||
fi
|
||||
|
||||
local env_active_profile="${OPS_ACTIVE_PROFILE:-}"
|
||||
local env_profile_names="${OPS_PROFILE_NAMES:-}"
|
||||
local env_report_root="${OPS_REPORT_ROOT:-}"
|
||||
local env_mainland_api_base_url="${OPS_MAINLAND_API_BASE_URL:-}"
|
||||
local env_overseas_api_base_url="${OPS_OVERSEAS_API_BASE_URL:-}"
|
||||
local env_control_plane_base_url="${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}"
|
||||
local env_requested_by="${OPS_DEFAULT_REQUESTED_BY:-}"
|
||||
local env_issued_by="${OPS_DEFAULT_ISSUED_BY:-}"
|
||||
local env_expires_in_hours="${OPS_DEFAULT_EXPIRES_IN_HOURS:-}"
|
||||
local env_root_dir="${OPS_DEFAULT_ROOT_DIR:-}"
|
||||
local env_node_agent_service_name="${OPS_NODE_AGENT_SERVICE_NAME:-}"
|
||||
local env_node_agent_env_file="${OPS_NODE_AGENT_ENV_FILE:-}"
|
||||
|
||||
OPS_CENTER_SCRIPT_DIR="${caller_dir}"
|
||||
OPS_CENTER_API_DIR="$(cd "${OPS_CENTER_SCRIPT_DIR}/../.." && pwd)"
|
||||
OPS_CENTER_PROJECT_ROOT="$(cd "${OPS_CENTER_API_DIR}/.." && pwd)"
|
||||
OPS_CENTER_RUNTIME_DIR="${OPS_CENTER_API_DIR}/runtime"
|
||||
OPS_CENTER_REPORT_ROOT_DEFAULT="${OPS_CENTER_RUNTIME_DIR}/ops-center-reports"
|
||||
OPS_CENTER_CONFIG_FILE="${DOMAINCHECK_OPS_CENTER_ENV:-${OPS_CENTER_ENV_FILE:-/etc/default/domaincheck-ops-center}}"
|
||||
|
||||
if [[ -f "${OPS_CENTER_CONFIG_FILE}" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
source "${OPS_CENTER_CONFIG_FILE}"
|
||||
fi
|
||||
|
||||
OPS_CENTER_ACTIVE_PROFILE="${OPS_CENTER_PROFILE:-${DOMAINCHECK_OPS_CENTER_PROFILE:-${env_active_profile:-${OPS_ACTIVE_PROFILE:-default}}}}"
|
||||
OPS_CENTER_ACTIVE_PROFILE_KEY="$(ops_center_normalize_profile_key "${OPS_CENTER_ACTIVE_PROFILE}")"
|
||||
OPS_CENTER_PROFILE_NAMES="${env_profile_names:-${OPS_PROFILE_NAMES:-${OPS_CENTER_ACTIVE_PROFILE}}}"
|
||||
OPS_CENTER_REPORT_ROOT="${env_report_root:-${OPS_REPORT_ROOT:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}}"
|
||||
OPS_CENTER_ENV_MAINLAND_API_BASE_URL="${env_mainland_api_base_url}"
|
||||
OPS_CENTER_ENV_OVERSEAS_API_BASE_URL="${env_overseas_api_base_url}"
|
||||
OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL="${env_control_plane_base_url}"
|
||||
OPS_CENTER_ENV_REQUESTED_BY="${env_requested_by}"
|
||||
OPS_CENTER_ENV_ISSUED_BY="${env_issued_by}"
|
||||
OPS_CENTER_ENV_EXPIRES_IN_HOURS="${env_expires_in_hours}"
|
||||
OPS_CENTER_ENV_ROOT_DIR="${env_root_dir}"
|
||||
OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME="${env_node_agent_service_name}"
|
||||
OPS_CENTER_ENV_NODE_AGENT_ENV_FILE="${env_node_agent_env_file}"
|
||||
}
|
||||
|
||||
ops_center_resolve_profile_value() {
|
||||
local suffix="$1"
|
||||
local fallback_value="${2:-}"
|
||||
local profile_key="${OPS_CENTER_ACTIVE_PROFILE_KEY:-DEFAULT}"
|
||||
local variable_name="OPS_PROFILE_${profile_key}_${suffix}"
|
||||
local profile_value="${!variable_name:-}"
|
||||
if [[ -n "${profile_value}" ]]; then
|
||||
printf '%s' "${profile_value}"
|
||||
else
|
||||
printf '%s' "${fallback_value}"
|
||||
fi
|
||||
}
|
||||
|
||||
ops_center_resolve_mainland_api_base_url() {
|
||||
ops_center_resolve_profile_value "MAINLAND_API_BASE_URL" "${OPS_CENTER_ENV_MAINLAND_API_BASE_URL:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_overseas_api_base_url() {
|
||||
ops_center_resolve_profile_value "OVERSEAS_API_BASE_URL" "${OPS_CENTER_ENV_OVERSEAS_API_BASE_URL:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_control_plane_base_url() {
|
||||
ops_center_resolve_profile_value "DEFAULT_CONTROL_PLANE_BASE_URL" "${OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_requested_by() {
|
||||
ops_center_resolve_profile_value "DEFAULT_REQUESTED_BY" "${OPS_CENTER_ENV_REQUESTED_BY:-${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_issued_by() {
|
||||
ops_center_resolve_profile_value "DEFAULT_ISSUED_BY" "${OPS_CENTER_ENV_ISSUED_BY:-${OPS_DEFAULT_ISSUED_BY:-ops-center}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_expires_in_hours() {
|
||||
ops_center_resolve_profile_value "DEFAULT_EXPIRES_IN_HOURS" "${OPS_CENTER_ENV_EXPIRES_IN_HOURS:-${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_root_dir() {
|
||||
ops_center_resolve_profile_value "DEFAULT_ROOT_DIR" "${OPS_CENTER_ENV_ROOT_DIR:-${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_node_agent_service_name() {
|
||||
ops_center_resolve_profile_value "NODE_AGENT_SERVICE_NAME" "${OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME:-${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}}"
|
||||
}
|
||||
|
||||
ops_center_resolve_node_agent_env_file() {
|
||||
ops_center_resolve_profile_value "NODE_AGENT_ENV_FILE" "${OPS_CENTER_ENV_NODE_AGENT_ENV_FILE:-${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}}"
|
||||
}
|
||||
|
||||
ops_center_ensure_report_dir() {
|
||||
local report_dir="${1:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}"
|
||||
mkdir -p "${report_dir}"
|
||||
}
|
||||
317
domain-api/deploy/multi-region/review_go_live_bundle.sh
Executable file
317
domain-api/deploy/multi-region/review_go_live_bundle.sh
Executable file
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_PATH="${1:-}"
|
||||
|
||||
if [[ -z "${TARGET_PATH}" ]]; then
|
||||
echo "usage: bash domain-api/deploy/multi-region/review_go_live_bundle.sh <manifest.json|report_dir>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d "${TARGET_PATH}" ]]; then
|
||||
MANIFEST_PATH="${TARGET_PATH%/}/manifest.json"
|
||||
else
|
||||
MANIFEST_PATH="${TARGET_PATH}"
|
||||
fi
|
||||
|
||||
if [[ ! -f "${MANIFEST_PATH}" ]]; then
|
||||
echo "manifest not found: ${MANIFEST_PATH}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - "${MANIFEST_PATH}" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
manifest_path = pathlib.Path(sys.argv[1])
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
|
||||
artifacts = payload.get("artifacts") or []
|
||||
failures = payload.get("failures") or []
|
||||
summary = payload.get("summary") or {}
|
||||
|
||||
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
|
||||
artifact_by_key = {str(item.get("key") or ""): item for item in artifacts if isinstance(item, dict)}
|
||||
critical_failures = [key for key in critical_keys if not bool((artifact_by_key.get(key) or {}).get("ok", False))]
|
||||
noncritical_failures = [key for key in failures if key not in critical_keys]
|
||||
env_audit_artifact = artifact_by_key.get("env_audit") or {}
|
||||
|
||||
env_audit_status = ""
|
||||
env_audit_headline = ""
|
||||
env_audit_missing_items = []
|
||||
env_audit_recommended_actions = []
|
||||
env_audit_blocking = False
|
||||
env_audit_attention = False
|
||||
env_audit_runtime_may_need_restart = False
|
||||
go_live_summary_payload = {}
|
||||
stack_diagnosis_payload = {}
|
||||
driver_feed_payload = {}
|
||||
codex_brief_payload = {}
|
||||
|
||||
def load_json_artifact(key: str) -> dict:
|
||||
artifact = artifact_by_key.get(key) or {}
|
||||
artifact_path = pathlib.Path(str(artifact.get("path") or ""))
|
||||
if not artifact_path.is_file():
|
||||
return {}
|
||||
try:
|
||||
loaded = json.loads(artifact_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
return loaded if isinstance(loaded, dict) else {}
|
||||
|
||||
env_audit_path = pathlib.Path(str(env_audit_artifact.get("path") or ""))
|
||||
if env_audit_path.is_file():
|
||||
raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace")
|
||||
marker = "[8/8] condensed env audit summary"
|
||||
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
|
||||
if candidate_text:
|
||||
try:
|
||||
env_payload = json.loads(candidate_text)
|
||||
except Exception:
|
||||
env_payload = {}
|
||||
env_audit_status = str(env_payload.get("status") or "")
|
||||
env_audit_headline = str(env_payload.get("headline") or "")
|
||||
env_audit_missing_items = list(env_payload.get("missing_items") or [])
|
||||
env_audit_recommended_actions = [str(item).strip() for item in list(env_payload.get("recommended_actions") or []) if str(item).strip()]
|
||||
env_audit_runtime_may_need_restart = bool(((env_payload.get("runtime") or {}).get("runtime_may_need_restart", False)))
|
||||
env_audit_blocking = env_audit_status == "blocked"
|
||||
env_audit_attention = env_audit_status == "attention"
|
||||
|
||||
go_live_summary_payload = load_json_artifact("go_live_summary")
|
||||
stack_diagnosis_payload = load_json_artifact("stack_diagnosis")
|
||||
driver_feed_payload = load_json_artifact("driver_feed")
|
||||
codex_brief_payload = load_json_artifact("codex_brief")
|
||||
|
||||
go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip()
|
||||
go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip()
|
||||
go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip()
|
||||
go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
|
||||
go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
|
||||
stack_diagnosis = stack_diagnosis_payload.get("diagnosis") or {}
|
||||
stack_launchpad_target_node_code = str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip()
|
||||
stack_launchpad_recovery_label = str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip()
|
||||
stack_launchpad_recovery_summary = str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip()
|
||||
stack_launchpad_bootstrap_pending_nodes = int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
|
||||
stack_launchpad_acceptance_ready_nodes = int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
|
||||
driver_feed_summary = driver_feed_payload.get("summary") or {}
|
||||
codex_brief_summary = codex_brief_payload.get("summary") or {}
|
||||
|
||||
|
||||
def nonempty_distinct(values: dict[str, str]) -> list[str]:
|
||||
return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()})
|
||||
|
||||
|
||||
def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]:
|
||||
return sorted({int(values[source]) for source in values if source in available_sources})
|
||||
|
||||
|
||||
launchpad_target_node_codes = {
|
||||
"go_live_summary": go_live_launchpad_target_node_code,
|
||||
"stack_diagnosis": stack_launchpad_target_node_code,
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||||
}
|
||||
launchpad_recovery_labels = {
|
||||
"go_live_summary": go_live_launchpad_recovery_label,
|
||||
"stack_diagnosis": stack_launchpad_recovery_label,
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||||
}
|
||||
launchpad_recovery_summaries = {
|
||||
"go_live_summary": go_live_launchpad_recovery_summary,
|
||||
"stack_diagnosis": stack_launchpad_recovery_summary,
|
||||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||||
}
|
||||
launchpad_bootstrap_pending_nodes = {
|
||||
"go_live_summary": go_live_launchpad_bootstrap_pending_nodes,
|
||||
"stack_diagnosis": stack_launchpad_bootstrap_pending_nodes,
|
||||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||||
}
|
||||
launchpad_acceptance_ready_nodes = {
|
||||
"go_live_summary": go_live_launchpad_acceptance_ready_nodes,
|
||||
"stack_diagnosis": stack_launchpad_acceptance_ready_nodes,
|
||||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||||
}
|
||||
|
||||
available_alignment_sources = {
|
||||
source
|
||||
for source, payload_value in {
|
||||
"go_live_summary": go_live_summary_payload,
|
||||
"stack_diagnosis": stack_diagnosis_payload,
|
||||
"driver_feed": driver_feed_payload,
|
||||
"codex_brief": codex_brief_payload,
|
||||
}.items()
|
||||
if isinstance(payload_value, dict) and payload_value
|
||||
}
|
||||
|
||||
launchpad_target_consistent = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_recovery_label_consistent = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_recovery_summary_consistent = len(nonempty_distinct({
|
||||
source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources
|
||||
})) <= 1
|
||||
launchpad_bootstrap_pending_consistent = len(distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources)) <= 1
|
||||
launchpad_acceptance_ready_consistent = len(distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources)) <= 1
|
||||
launchpad_alignment_consistent = all([
|
||||
launchpad_target_consistent,
|
||||
launchpad_recovery_label_consistent,
|
||||
launchpad_recovery_summary_consistent,
|
||||
launchpad_bootstrap_pending_consistent,
|
||||
launchpad_acceptance_ready_consistent,
|
||||
])
|
||||
|
||||
node_agent_env_missing = any(
|
||||
str(item).strip() == "missing_env:/etc/default/domaincheck-node-agent"
|
||||
for item in env_audit_missing_items
|
||||
)
|
||||
|
||||
if critical_failures or env_audit_blocking:
|
||||
status = "blocked"
|
||||
if critical_failures and env_audit_blocking:
|
||||
headline = "关键报告存在缺失,且环境审计已给出阻断结论,当前不适合上线。"
|
||||
elif critical_failures:
|
||||
headline = "上线证据包仍缺关键报告,暂不建议直接宣称收口完成。"
|
||||
else:
|
||||
headline = "环境审计已给出阻断结论,需先修复现场环境后再进入发布门禁。"
|
||||
elif noncritical_failures or env_audit_attention or not launchpad_alignment_consistent:
|
||||
status = "attention"
|
||||
if not launchpad_alignment_consistent:
|
||||
headline = "核心报告已齐,但 launchpad 摘要在不同 surface 之间存在漂移,建议先复核再发版。"
|
||||
elif noncritical_failures and env_audit_attention:
|
||||
headline = "核心报告已齐,但补充报告和环境审计都提示仍有缺口,建议先复核再发版。"
|
||||
elif noncritical_failures:
|
||||
headline = "核心报告已齐,但仍有补充报告超时或失败,建议先复核再发版。"
|
||||
else:
|
||||
headline = "核心报告已齐,但环境审计仍提示基础缺口,建议补齐后再发版。"
|
||||
else:
|
||||
status = "ready"
|
||||
headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
|
||||
|
||||
result = {
|
||||
"report_dir": str(payload.get("report_dir") or manifest_path.parent),
|
||||
"status": status,
|
||||
"headline": headline,
|
||||
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
|
||||
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
|
||||
"critical_failures": critical_failures,
|
||||
"noncritical_failures": noncritical_failures,
|
||||
"env_audit": {
|
||||
"ok": bool(env_audit_artifact.get("ok", False)),
|
||||
"status": env_audit_status,
|
||||
"headline": env_audit_headline,
|
||||
"missing_items": env_audit_missing_items,
|
||||
"runtime_may_need_restart": env_audit_runtime_may_need_restart,
|
||||
"recommended_actions": env_audit_recommended_actions,
|
||||
},
|
||||
"go_live_summary": {
|
||||
"go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(),
|
||||
"publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(),
|
||||
"operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(),
|
||||
"next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(),
|
||||
"launchpad_recommended_target_node_code": go_live_launchpad_target_node_code,
|
||||
"launchpad_recommended_recovery_label": go_live_launchpad_recovery_label,
|
||||
"launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary,
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes,
|
||||
},
|
||||
"stack_diagnosis": {
|
||||
"stack_status": str((stack_diagnosis_payload.get("diagnosis") or {}).get("stack_status") or "").strip(),
|
||||
"operator_title": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("operator_decision") or {}).get("title") or "").strip(),
|
||||
"next_step_action_code": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("next_step") or {}).get("action_code") or "").strip(),
|
||||
"launchpad_recommended_target_node_code": stack_launchpad_target_node_code,
|
||||
"launchpad_recommended_recovery_label": stack_launchpad_recovery_label,
|
||||
"launchpad_recommended_recovery_summary": stack_launchpad_recovery_summary,
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": stack_launchpad_bootstrap_pending_nodes,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": stack_launchpad_acceptance_ready_nodes,
|
||||
},
|
||||
"driver_feed": {
|
||||
"headline": str(driver_feed_payload.get("headline") or "").strip(),
|
||||
"launch_status": str(((driver_feed_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(),
|
||||
"launch_ready": bool(((driver_feed_payload.get("automation_coverage") or {}).get("launch_ready", False))),
|
||||
"launchpad_recommended_target_node_code": str(
|
||||
((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "")
|
||||
).strip(),
|
||||
"launchpad_recommended_recovery_label": str(
|
||||
((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "")
|
||||
).strip(),
|
||||
},
|
||||
"codex_brief": {
|
||||
"headline": str(codex_brief_payload.get("headline") or "").strip(),
|
||||
"focus_entry_key": str(((codex_brief_payload.get("focus") or {}).get("entry_key") or "")).strip(),
|
||||
"launch_status": str(((codex_brief_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(),
|
||||
"launch_ready": bool(((codex_brief_payload.get("automation_coverage") or {}).get("launch_ready", False))),
|
||||
"launchpad_recommended_target_node_code": str(
|
||||
((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "")
|
||||
).strip(),
|
||||
"launchpad_recommended_recovery_label": str(
|
||||
((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "")
|
||||
).strip(),
|
||||
},
|
||||
"launchpad_alignment": {
|
||||
"consistent": launchpad_alignment_consistent,
|
||||
"target_node_code_consistent": launchpad_target_consistent,
|
||||
"recovery_label_consistent": launchpad_recovery_label_consistent,
|
||||
"recovery_summary_consistent": launchpad_recovery_summary_consistent,
|
||||
"bootstrap_pending_consistent": launchpad_bootstrap_pending_consistent,
|
||||
"acceptance_ready_consistent": launchpad_acceptance_ready_consistent,
|
||||
"available_sources": sorted(available_alignment_sources),
|
||||
"target_node_codes": launchpad_target_node_codes,
|
||||
"recovery_labels": launchpad_recovery_labels,
|
||||
"recovery_summaries": launchpad_recovery_summaries,
|
||||
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
|
||||
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
|
||||
},
|
||||
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
|
||||
"recommended_next_steps": [
|
||||
"先看 00_env_audit.txt,确认当前机器环境是否存在阻断或注意项",
|
||||
"先看 02_go_live_summary.json 和 03_stack_diagnosis.json",
|
||||
"再看 04_driver_feed.json 和 05_codex_brief.json",
|
||||
"若涉及发布门禁,再看 06_release_launchpad.json",
|
||||
"若 manifest 里仍有失败项,再看对应 artifact 原文件",
|
||||
]
|
||||
+ (
|
||||
[
|
||||
"当前缺口指向 Node Agent 未接管:先执行 agent-gap-export,拿到首个缺口节点的接管包",
|
||||
"随后执行 node-bootstrap-plan,为缺口节点生成可直接落地的接管脚本",
|
||||
"如果仓库代码已更新但 bootstrap-plan 输出仍偏旧,先重启控制面 API 再重新导出接管计划",
|
||||
]
|
||||
if node_agent_env_missing
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
[
|
||||
"env-audit 已识别运行中 API 可能仍是旧代码:先执行 drive_ops_center.sh runtime-refresh-recover,按固定链路完成 API 重启与复检",
|
||||
"若仍需人工逐条确认,再执行 recommended_actions 里的 API 重启与复检命令"
|
||||
]
|
||||
if env_audit_runtime_may_need_restart
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
[
|
||||
f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code},建议优先执行:{go_live_launchpad_recovery_label or '节点恢复'}",
|
||||
"先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段,确认三处摘要是否一致",
|
||||
(go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空,再回到 06_release_launchpad.json 查看 gap rows 详情"),
|
||||
]
|
||||
if go_live_launchpad_target_node_code
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
[
|
||||
"当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致,先不要直接交付。",
|
||||
"优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json,再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。",
|
||||
"若只有局部摘要偏旧,先执行 drive_ops_center.sh runtime-refresh-recover,并重新导出 go-live bundle。",
|
||||
]
|
||||
if not launchpad_alignment_consistent
|
||||
else []
|
||||
),
|
||||
}
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
PY
|
||||
@@ -0,0 +1,15 @@
|
||||
OPS_CONTROL_PLANE_BASE_URL=http://127.0.0.1:8100
|
||||
OPS_AGENT_TOKEN=
|
||||
OPS_AGENT_POLL_INTERVAL_SECONDS=5
|
||||
|
||||
NODE_CODE=mainland-worker-01
|
||||
NODE_REGION=mainland
|
||||
NODE_ROLE=worker
|
||||
|
||||
WORKER_SERVICE_NAME=domaincheck-worker
|
||||
API_SERVICE_NAME=domaincheck-api
|
||||
SYNC_AGENT_SERVICE_NAME=domaincheck-sync-agent
|
||||
NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
|
||||
|
||||
OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]
|
||||
OPS_AGENT_LABELS={}
|
||||
@@ -0,0 +1,49 @@
|
||||
# Overseas control-plane unified entrypoint config
|
||||
#
|
||||
# Recommended target path:
|
||||
# /etc/default/domaincheck-ops-center
|
||||
#
|
||||
# Usage:
|
||||
# export DOMAINCHECK_OPS_CENTER_ENV=/etc/default/domaincheck-ops-center
|
||||
# bash domain-api/deploy/multi-region/drive_ops_center.sh doctor
|
||||
|
||||
# Active profile name
|
||||
OPS_ACTIVE_PROFILE=default
|
||||
|
||||
# Optional profile list shown by `drive_ops_center.sh config`
|
||||
# OPS_PROFILE_NAMES=default,prod,staging
|
||||
|
||||
# Mainland controller API base URL
|
||||
OPS_MAINLAND_API_BASE_URL=http://127.0.0.1:8100
|
||||
|
||||
# Overseas control-plane API base URL
|
||||
OPS_OVERSEAS_API_BASE_URL=http://127.0.0.1:8100
|
||||
|
||||
# Public control-plane base URL used in generated node-agent bootstrap plans
|
||||
OPS_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com
|
||||
|
||||
# Default request identity used by CLI wrappers
|
||||
OPS_DEFAULT_REQUESTED_BY=cli/ops-center
|
||||
|
||||
# Default metadata used when generating node-agent bootstrap plans
|
||||
OPS_DEFAULT_ISSUED_BY=ops-center
|
||||
OPS_DEFAULT_EXPIRES_IN_HOURS=72
|
||||
OPS_DEFAULT_ROOT_DIR=/opt/domaincheck
|
||||
|
||||
# Optional report export root
|
||||
# OPS_REPORT_ROOT=/opt/domaincheck/domain-api/runtime/ops-center-reports
|
||||
|
||||
# Local node-agent status check defaults
|
||||
OPS_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
|
||||
OPS_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent
|
||||
|
||||
# Optional per-profile overrides:
|
||||
# OPS_PROFILE_PROD_MAINLAND_API_BASE_URL=http://10.0.0.10:8100
|
||||
# OPS_PROFILE_PROD_OVERSEAS_API_BASE_URL=https://api.example.com
|
||||
# OPS_PROFILE_PROD_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com
|
||||
# OPS_PROFILE_PROD_DEFAULT_REQUESTED_BY=cli/ops-center-prod
|
||||
# OPS_PROFILE_PROD_DEFAULT_ISSUED_BY=ops-center-prod
|
||||
# OPS_PROFILE_PROD_DEFAULT_EXPIRES_IN_HOURS=72
|
||||
# OPS_PROFILE_PROD_DEFAULT_ROOT_DIR=/opt/domaincheck
|
||||
# OPS_PROFILE_PROD_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent
|
||||
# OPS_PROFILE_PROD_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent
|
||||
17
domain-api/deploy/systemd/domain-node-agent.service
Normal file
17
domain-api/deploy/systemd/domain-node-agent.service
Normal file
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=domainCheck Node Agent
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=/opt/domaincheck/domain-api
|
||||
EnvironmentFile=-/etc/default/domaincheck-node-agent
|
||||
Environment="PATH=/home/www/.nvm/versions/node/v20.20.2/bin:/home/www/.local/bin:/home/www/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
|
||||
ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m app.node_agent
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
User=www
|
||||
Group=www
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
98
domain-api/tests/test_build_info_service.py
Normal file
98
domain-api/tests/test_build_info_service.py
Normal file
@@ -0,0 +1,98 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.build_info_service import (
|
||||
_expected_route_paths,
|
||||
get_runtime_build_info,
|
||||
remember_registered_route_paths,
|
||||
)
|
||||
|
||||
|
||||
class BuildInfoServiceTests(unittest.TestCase):
|
||||
def tearDown(self) -> None:
|
||||
remember_registered_route_paths([])
|
||||
|
||||
def test_runtime_build_info_prefers_configured_manifest_and_reports_registered_surface(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
manifest_path = Path(temp_dir) / "release_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": "domaincheck_release_demo",
|
||||
"generated_at": "2026-04-18T12:00:00",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"commit_ref": "main",
|
||||
"sha256": "demo-sha256",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
route_paths = list(_expected_route_paths().values())
|
||||
remember_registered_route_paths(route_paths)
|
||||
|
||||
with patch.multiple(
|
||||
settings,
|
||||
build_manifest_path=str(manifest_path),
|
||||
build_commit_sha="",
|
||||
build_commit_ref="",
|
||||
build_generated_at="",
|
||||
build_package_name="",
|
||||
build_checksum="",
|
||||
build_source_label="",
|
||||
):
|
||||
payload = get_runtime_build_info()
|
||||
|
||||
self.assertEqual("configured_manifest", payload["source"])
|
||||
self.assertEqual("domaincheck_release_demo", payload["package_name"])
|
||||
self.assertEqual("bd6bcb2", payload["commit_sha"])
|
||||
self.assertEqual("main", payload["commit_ref"])
|
||||
self.assertEqual("demo-sha256", payload["checksum"])
|
||||
self.assertEqual(str(manifest_path), payload["manifest_path"])
|
||||
self.assertTrue(payload["route_surface"]["surface_complete"])
|
||||
self.assertEqual([], payload["route_surface"]["missing_keys"])
|
||||
self.assertEqual("registered", payload["route_surface"]["mode"])
|
||||
self.assertIn("repository_capabilities", payload)
|
||||
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
|
||||
self.assertIn("supports_multi_layout_bootstrap", payload["repository_capabilities"])
|
||||
|
||||
def test_runtime_build_info_env_override_wins_and_missing_routes_are_visible(self) -> None:
|
||||
with patch.multiple(
|
||||
settings,
|
||||
build_manifest_path="",
|
||||
build_commit_sha="246838ae4c07",
|
||||
build_commit_ref="main",
|
||||
build_generated_at="2026-04-18T12:10:00",
|
||||
build_package_name="domaincheck_release_env",
|
||||
build_checksum="env-sha256",
|
||||
build_source_label="env-override",
|
||||
):
|
||||
payload = get_runtime_build_info(route_paths=[_expected_route_paths()["ops_contracts"]])
|
||||
|
||||
self.assertEqual("env-override", payload["source"])
|
||||
self.assertEqual("domaincheck_release_env", payload["package_name"])
|
||||
self.assertEqual("246838ae4c07", payload["commit_sha"])
|
||||
self.assertEqual("main", payload["commit_ref"])
|
||||
self.assertEqual("env-sha256", payload["checksum"])
|
||||
self.assertFalse(payload["route_surface"]["surface_complete"])
|
||||
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_recovery_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_recovery_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_scene_log", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("runtime_build_info", payload["route_surface"]["missing_keys"])
|
||||
self.assertEqual("registered", payload["route_surface"]["mode"])
|
||||
self.assertIn("repository_capabilities", payload)
|
||||
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
111
domain-api/tests/test_drive_ops_center_docs.py
Normal file
111
domain-api/tests/test_drive_ops_center_docs.py
Normal file
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DriveOpsCenterDocsTests(unittest.TestCase):
|
||||
def test_authoritative_docs_only_reference_supported_commands(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
|
||||
doc_paths = [
|
||||
repo_root / "domain-api" / "deploy" / "multi-region" / "README.md",
|
||||
repo_root / "docs" / "25_domainCheck_海外单脑控制面上线收口总表.md",
|
||||
repo_root / "docs" / "26_domainCheck_发布前运行验证与交付模板.md",
|
||||
]
|
||||
|
||||
help_output = subprocess.run(
|
||||
["bash", str(script_path), "help"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
|
||||
supported_commands = self._extract_supported_commands(help_output)
|
||||
documented_commands = self._extract_documented_commands(doc_paths)
|
||||
|
||||
missing_commands = sorted(documented_commands - supported_commands)
|
||||
self.assertEqual(
|
||||
[],
|
||||
missing_commands,
|
||||
msg=(
|
||||
"These drive_ops_center.sh commands are referenced by authoritative docs "
|
||||
"but missing from the CLI help output: "
|
||||
+ ", ".join(missing_commands)
|
||||
),
|
||||
)
|
||||
self.assertIn("stack-diagnosis", documented_commands)
|
||||
self.assertIn("stack-diagnosis", supported_commands)
|
||||
|
||||
def test_stack_diagnosis_summary_argument_normalization_does_not_treat_summary_as_host(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_OPS_CENTER_ENV"] = "/tmp/domaincheck-ops-center-does-not-exist"
|
||||
env["OPS_MAINLAND_API_BASE_URL"] = "http://127.0.0.1:9"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path), "stack-diagnosis", "summary"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
combined_output = f"{result.stdout}\n{result.stderr}"
|
||||
self.assertNotIn("Could not resolve host: summary", combined_output)
|
||||
self.assertNotIn("NameError", combined_output)
|
||||
self.assertIn("[9/9] condensed stack summary", combined_output)
|
||||
|
||||
def test_env_drift_recommends_stack_diagnosis_instead_of_stack_check(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "check_env_drift.sh"
|
||||
script_text = script_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn(
|
||||
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")',
|
||||
script_text,
|
||||
)
|
||||
self.assertNotIn(
|
||||
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-check http://127.0.0.1:8100 summary")',
|
||||
script_text,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_supported_commands(help_output: str) -> set[str]:
|
||||
commands: set[str] = set()
|
||||
in_commands_section = False
|
||||
for raw_line in help_output.splitlines():
|
||||
line = raw_line.rstrip()
|
||||
if line.strip() == "commands:":
|
||||
in_commands_section = True
|
||||
continue
|
||||
if line.strip() == "examples:":
|
||||
break
|
||||
if not in_commands_section:
|
||||
continue
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
command = stripped.split()[0]
|
||||
if re.fullmatch(r"[a-z0-9-]+", command):
|
||||
commands.add(command)
|
||||
return commands
|
||||
|
||||
@staticmethod
|
||||
def _extract_documented_commands(doc_paths: list[Path]) -> set[str]:
|
||||
pattern = re.compile(r"drive_ops_center\.sh\s+([a-z0-9-]+)")
|
||||
commands: set[str] = set()
|
||||
for doc_path in doc_paths:
|
||||
text = doc_path.read_text(encoding="utf-8")
|
||||
commands.update(match.group(1) for match in pattern.finditer(text))
|
||||
return commands
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
284
domain-api/tests/test_node_agent_delivery_queue.py
Normal file
284
domain-api/tests/test_node_agent_delivery_queue.py
Normal file
@@ -0,0 +1,284 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from unittest.mock import patch
|
||||
|
||||
from app import node_agent
|
||||
|
||||
|
||||
class NodeAgentDeliveryQueueTests(unittest.TestCase):
|
||||
def test_base_payload_includes_delivery_queue_snapshot(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
node_agent._ensure_queue_dirs()
|
||||
payload = node_agent._base_payload()
|
||||
|
||||
delivery_queue = payload["metadata"]["delivery_queue"]
|
||||
self.assertEqual("healthy", delivery_queue["state"])
|
||||
self.assertEqual(0, delivery_queue["pending_count"])
|
||||
self.assertEqual(0, delivery_queue["dead_letter_count"])
|
||||
|
||||
def test_job_event_network_failure_is_queued_for_retry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
side_effect=urllib.error.URLError("offline"),
|
||||
):
|
||||
result = node_agent._job_event(
|
||||
7,
|
||||
event_type="executor_received",
|
||||
message="accepted",
|
||||
client_event_id="evt-001",
|
||||
summary_text="已接单",
|
||||
focus_ref={"kind": "ops_job", "job_id": 7},
|
||||
)
|
||||
|
||||
self.assertEqual("queued", result["state"])
|
||||
pending_dir = os.path.join(temp_dir, "pending")
|
||||
pending_files = sorted(os.listdir(pending_dir))
|
||||
self.assertEqual(1, len(pending_files))
|
||||
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
|
||||
record = json.load(handle)
|
||||
self.assertEqual("evt-001", record["payload"]["client_event_id"])
|
||||
self.assertEqual("已接单", record["payload"]["summary_text"])
|
||||
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
|
||||
self.assertEqual("job_event", record["kind"])
|
||||
|
||||
def test_job_complete_transport_failure_preserves_client_request_id(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
side_effect=urllib.error.URLError("offline"),
|
||||
):
|
||||
result = node_agent._job_complete(
|
||||
9,
|
||||
status="success",
|
||||
stdout="done",
|
||||
stderr="",
|
||||
result={"ok": True},
|
||||
client_request_id="complete-001",
|
||||
duration_ms=321,
|
||||
summary_text="执行成功",
|
||||
focus_ref={"kind": "ops_job", "job_id": 9},
|
||||
)
|
||||
|
||||
self.assertEqual("queued", result["state"])
|
||||
pending_dir = os.path.join(temp_dir, "pending")
|
||||
pending_files = sorted(os.listdir(pending_dir))
|
||||
self.assertEqual(1, len(pending_files))
|
||||
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
|
||||
record = json.load(handle)
|
||||
self.assertEqual("complete-001", record["payload"]["client_request_id"])
|
||||
self.assertEqual(321, record["payload"]["duration_ms"])
|
||||
self.assertEqual("执行成功", record["payload"]["summary_text"])
|
||||
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
|
||||
self.assertEqual("job_complete", record["kind"])
|
||||
|
||||
def test_flush_delivery_queue_delivers_pending_records(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_event",
|
||||
"/api/v1/ops/agent/jobs/7/events",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "executor_received",
|
||||
"message": "accepted",
|
||||
"client_event_id": "evt-002",
|
||||
},
|
||||
"evt-002",
|
||||
)
|
||||
node_agent._store_pending_delivery(record)
|
||||
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
|
||||
summary = node_agent._flush_delivery_queue(limit=10)
|
||||
|
||||
self.assertEqual(1, summary["delivered"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
|
||||
def test_flush_delivery_queue_moves_semantic_failure_to_dead_letter(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_complete",
|
||||
"/api/v1/ops/agent/jobs/9/complete",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "success",
|
||||
"client_request_id": "complete-002",
|
||||
},
|
||||
"complete-002",
|
||||
)
|
||||
node_agent._store_pending_delivery(record)
|
||||
with patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
return_value={
|
||||
"code": 1,
|
||||
"message": "任务不存在或不属于当前节点",
|
||||
"detail_code": "ops_job_not_owned_by_agent",
|
||||
},
|
||||
):
|
||||
summary = node_agent._flush_delivery_queue(limit=10)
|
||||
|
||||
self.assertEqual(1, summary["dead_letter"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
self.assertEqual(1, len(os.listdir(os.path.join(temp_dir, "dead-letter"))))
|
||||
|
||||
def test_replay_dead_letter_records_moves_record_back_to_pending_and_flushes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_event",
|
||||
"/api/v1/ops/agent/jobs/7/events",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "executor_received",
|
||||
"message": "accepted",
|
||||
"client_event_id": "evt-003",
|
||||
},
|
||||
"evt-003",
|
||||
)
|
||||
node_agent._store_dead_letter_delivery(
|
||||
record,
|
||||
reason="temporary failure",
|
||||
detail_code="transient_error",
|
||||
)
|
||||
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
|
||||
ok, message, data = node_agent._execute_action(
|
||||
"delivery.queue.replay",
|
||||
{
|
||||
"selector": {"record_id": "evt-003"},
|
||||
"limit": 10,
|
||||
"flush_after_replay": True,
|
||||
"reason": "恢复后重放",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已重放 1 条死信记录", message)
|
||||
self.assertEqual(1, data["replayed_total"])
|
||||
self.assertEqual(1, data["flush_summary"]["delivered"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
|
||||
def test_discard_dead_letter_records_archives_record(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_complete",
|
||||
"/api/v1/ops/agent/jobs/8/complete",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "failed",
|
||||
"client_request_id": "complete-003",
|
||||
},
|
||||
"complete-003",
|
||||
)
|
||||
node_agent._store_dead_letter_delivery(
|
||||
record,
|
||||
reason="permanent failure",
|
||||
detail_code="ops_job_invalid_status",
|
||||
)
|
||||
ok, message, data = node_agent._execute_action(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"selector": {"record_id": "complete-003"},
|
||||
"limit": 10,
|
||||
"discarded_by": "web-ui",
|
||||
"reason": "确认不需要再补发",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已丢弃 1 条死信记录", message)
|
||||
self.assertEqual(1, data["discarded_total"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
|
||||
discarded_files = os.listdir(os.path.join(temp_dir, "discarded"))
|
||||
self.assertEqual(1, len(discarded_files))
|
||||
with open(os.path.join(temp_dir, "discarded", discarded_files[0]), "r", encoding="utf-8") as handle:
|
||||
archived = json.load(handle)
|
||||
self.assertEqual("web-ui", archived["discarded_by"])
|
||||
self.assertEqual("确认不需要再补发", archived["discard_reason"])
|
||||
|
||||
def test_process_job_passes_envelope_context_to_completion(self) -> None:
|
||||
job = {
|
||||
"id": 12,
|
||||
"job_code": "ops-20260418-000012",
|
||||
"job_type": "release_deploy",
|
||||
"action": "logs.collect",
|
||||
"payload": {"service_name": "domaincheck-worker", "lines": 120},
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"focus_ref": {"kind": "ops_job", "job_id": 12, "job_code": "ops-20260418-000012"},
|
||||
"release_context": {"release_version": "v2026.04.18", "rollout_id": 3},
|
||||
"step_ref": {"step_id": 22, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
|
||||
}
|
||||
with patch.object(node_agent, "_job_start") as mock_job_start, patch.object(
|
||||
node_agent,
|
||||
"_job_event",
|
||||
) as mock_job_event, patch.object(
|
||||
node_agent,
|
||||
"_execute_action",
|
||||
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
|
||||
) as mock_execute_action, patch.object(
|
||||
node_agent,
|
||||
"_job_complete",
|
||||
return_value={"state": "sent"},
|
||||
) as mock_job_complete:
|
||||
node_agent._process_job(job)
|
||||
|
||||
self.assertEqual("worker_logs", mock_job_start.call_args.kwargs["job"]["step_key"])
|
||||
self.assertEqual("ops_job", mock_job_event.call_args.kwargs["focus_ref"]["kind"])
|
||||
self.assertEqual("已接单 logs.collect", mock_job_event.call_args.kwargs["summary_text"])
|
||||
self.assertEqual("logs.collect", mock_execute_action.call_args.args[0])
|
||||
self.assertEqual("v2026.04.18", mock_job_complete.call_args.kwargs["release_context"]["release_version"])
|
||||
self.assertEqual("ops_job", mock_job_complete.call_args.kwargs["focus_ref"]["kind"])
|
||||
self.assertEqual("收集完成", mock_job_complete.call_args.kwargs["summary_text"])
|
||||
self.assertGreaterEqual(mock_job_complete.call_args.kwargs["duration_ms"], 0)
|
||||
|
||||
def test_process_job_continues_when_job_start_delivery_fails(self) -> None:
|
||||
job = {
|
||||
"id": 18,
|
||||
"job_code": "ops-20260418-000018",
|
||||
"job_type": "ops_action",
|
||||
"action": "logs.collect",
|
||||
"payload": {"service_name": "domaincheck-worker", "lines": 50},
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"focus_ref": {"kind": "ops_job", "job_id": 18, "job_code": "ops-20260418-000018"},
|
||||
"release_context": {},
|
||||
"step_ref": {"step_id": 28, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
|
||||
}
|
||||
with patch.object(
|
||||
node_agent,
|
||||
"_job_start",
|
||||
side_effect=urllib.error.URLError("temporary offline"),
|
||||
) as mock_job_start, patch.object(
|
||||
node_agent,
|
||||
"_job_event",
|
||||
return_value={"state": "queued"},
|
||||
) as mock_job_event, patch.object(
|
||||
node_agent,
|
||||
"_execute_action",
|
||||
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
|
||||
) as mock_execute_action, patch.object(
|
||||
node_agent,
|
||||
"_job_complete",
|
||||
return_value={"state": "sent"},
|
||||
) as mock_job_complete:
|
||||
node_agent._process_job(job)
|
||||
|
||||
mock_job_start.assert_called_once()
|
||||
mock_execute_action.assert_called_once()
|
||||
mock_job_complete.assert_called_once()
|
||||
event_payload = mock_job_event.call_args.kwargs["payload"]
|
||||
self.assertEqual("failed_local", event_payload["start_delivery_state"])
|
||||
self.assertIn("temporary offline", event_payload["start_delivery_error"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
41
domain-api/tests/test_node_agent_release_normalization.py
Normal file
41
domain-api/tests/test_node_agent_release_normalization.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
from app import node_agent
|
||||
|
||||
|
||||
class NodeAgentReleaseNormalizationTests(unittest.TestCase):
|
||||
def test_normalize_text_list_supports_strings_and_lists(self) -> None:
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker"],
|
||||
node_agent._normalize_text_list("domaincheck-api\ndomaincheck-worker"),
|
||||
)
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker"],
|
||||
node_agent._normalize_text_list(["domaincheck-api", " domaincheck-worker "]),
|
||||
)
|
||||
self.assertEqual([], node_agent._normalize_text_list(""))
|
||||
|
||||
def test_health_check_services_fall_back_to_restart_services_when_missing(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual(["domaincheck-worker"], result)
|
||||
|
||||
def test_health_check_services_can_be_explicitly_empty(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{"health_check_services": ""},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual([], result)
|
||||
|
||||
def test_health_check_services_accept_multiline_string(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{"health_check_services": "domaincheck-api\ndomaincheck-sync-agent"},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-sync-agent"], result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
115
domain-api/tests/test_ops_action_executor_core.py
Normal file
115
domain-api/tests/test_ops_action_executor_core.py
Normal file
@@ -0,0 +1,115 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_action_executor_core import build_service_name_map, execute_structured_action
|
||||
|
||||
|
||||
class _FakeRunner:
|
||||
def __init__(self, responses: dict[tuple[str, ...], tuple[int, str, str]]) -> None:
|
||||
self.responses = responses
|
||||
self.calls: list[tuple[str, ...]] = []
|
||||
|
||||
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
|
||||
key = tuple(command)
|
||||
self.calls.append(key)
|
||||
return self.responses.get(key, (0, "", ""))
|
||||
|
||||
|
||||
class OpsActionExecutorCoreTests(unittest.TestCase):
|
||||
def test_health_snapshot_includes_host_context_and_ok_flags(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("systemctl", "is-active", "domaincheck-api"): (0, "active", ""),
|
||||
("systemctl", "is-active", "domaincheck-worker"): (3, "inactive", ""),
|
||||
("systemctl", "is-active", "domaincheck-sync-agent"): (0, "active", ""),
|
||||
("systemctl", "is-active", "domaincheck-node-agent"): (0, "active", ""),
|
||||
}
|
||||
)
|
||||
|
||||
ok, message, data = execute_structured_action(
|
||||
"health.snapshot",
|
||||
{},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
host_context={"hostname": "mainland-controller-01", "ip": "121.204.244.188"},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("health snapshot collected", message)
|
||||
self.assertEqual("mainland-controller-01", data["hostname"])
|
||||
self.assertEqual("121.204.244.188", data["ip"])
|
||||
self.assertTrue(data["checks"]["api"]["ok"])
|
||||
self.assertFalse(data["checks"]["worker"]["ok"])
|
||||
self.assertEqual(4, len(runner.calls))
|
||||
|
||||
def test_logs_collect_defaults_to_worker_service_and_clamps_lines(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"): (
|
||||
0,
|
||||
"line-1\nline-2",
|
||||
"",
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
ok, message, data = execute_structured_action(
|
||||
"logs.collect",
|
||||
{"lines": 9999},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("line-1\nline-2", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual(500, data["lines"])
|
||||
self.assertEqual(
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"),
|
||||
runner.calls[0],
|
||||
)
|
||||
|
||||
def test_logs_collect_falls_back_to_worker_log_file_when_journal_is_denied(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "120", "--no-pager"): (
|
||||
1,
|
||||
"",
|
||||
"No journal files were opened due to insufficient permissions.",
|
||||
)
|
||||
}
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
log_path = Path(temp_dir) / "detect_worker.log"
|
||||
log_path.write_text("worker-a\nworker-b\n", encoding="utf-8")
|
||||
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
|
||||
ok, message, data = execute_structured_action(
|
||||
"logs.collect",
|
||||
{"lines": 120},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("worker-a\nworker-b", message)
|
||||
self.assertEqual("file_fallback", data["collection_source"])
|
||||
self.assertTrue(data["fallback_used"])
|
||||
self.assertIn("detect_worker.log", data["log_paths"][0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1181
domain-api/tests/test_ops_agent_service.py
Normal file
1181
domain-api/tests/test_ops_agent_service.py
Normal file
File diff suppressed because it is too large
Load Diff
96
domain-api/tests/test_ops_api_routes.py
Normal file
96
domain-api/tests/test_ops_api_routes.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.api.routes.ops import (
|
||||
ops_doctor_decision,
|
||||
ops_go_live_bundle,
|
||||
ops_go_live_review,
|
||||
ops_go_live_signoff,
|
||||
ops_go_live_summary,
|
||||
)
|
||||
|
||||
|
||||
class OpsApiRoutesTestCase(unittest.TestCase):
|
||||
@patch("app.api.routes.ops.get_ops_go_live_signoff")
|
||||
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
|
||||
mock_get_ops_go_live_signoff.return_value = {
|
||||
"signoff_status": "ready",
|
||||
"headline": "ok",
|
||||
}
|
||||
|
||||
response = ops_go_live_signoff(base_url="http://127.0.0.1:8100", report_dir="/tmp/signoff")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("ready", response.data["signoff_status"])
|
||||
self.assertEqual("ok", response.data["headline"])
|
||||
mock_get_ops_go_live_signoff.assert_called_once_with(
|
||||
base_url="http://127.0.0.1:8100",
|
||||
report_dir="/tmp/signoff",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_summary")
|
||||
def test_go_live_summary_route_passes_base_url(self, mock_get_ops_go_live_summary) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
}
|
||||
|
||||
response = ops_go_live_summary(base_url="http://example.test")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["go_live_status"])
|
||||
mock_get_ops_go_live_summary.assert_called_once_with(base_url="http://example.test")
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_bundle")
|
||||
def test_go_live_bundle_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_bundle) -> None:
|
||||
mock_get_ops_go_live_bundle.return_value = {
|
||||
"status": "missing",
|
||||
}
|
||||
|
||||
response = ops_go_live_bundle(base_url="http://example.test", report_dir="/tmp/go-live")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("missing", response.data["status"])
|
||||
mock_get_ops_go_live_bundle.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/go-live",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_review")
|
||||
def test_go_live_review_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_review) -> None:
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "review",
|
||||
}
|
||||
|
||||
response = ops_go_live_review(base_url="http://example.test", report_dir="/tmp/review")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["status"])
|
||||
self.assertEqual("review", response.data["headline"])
|
||||
mock_get_ops_go_live_review.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/review",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_doctor_decision")
|
||||
def test_doctor_decision_route_passes_base_url_and_report_dir(self, mock_get_ops_doctor_decision) -> None:
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "doctor",
|
||||
}
|
||||
|
||||
response = ops_doctor_decision(base_url="http://example.test", report_dir="/tmp/doctor")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["status"])
|
||||
self.assertEqual("doctor", response.data["headline"])
|
||||
mock_get_ops_doctor_decision.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/doctor",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
44
domain-api/tests/test_ops_command_service.py
Normal file
44
domain-api/tests/test_ops_command_service.py
Normal file
@@ -0,0 +1,44 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_command_service import build_bash_command, ops_script_path
|
||||
|
||||
|
||||
class OpsCommandServiceTests(unittest.TestCase):
|
||||
def test_ops_script_path_prefixes_multi_region_directory(self) -> None:
|
||||
self.assertEqual(
|
||||
"domain-api/deploy/multi-region/drive_ops_center.sh",
|
||||
ops_script_path("drive_ops_center.sh"),
|
||||
)
|
||||
|
||||
def test_ops_script_path_normalizes_leading_slash(self) -> None:
|
||||
self.assertEqual(
|
||||
"domain-api/deploy/multi-region/check_ops_center_stack.sh",
|
||||
ops_script_path("/check_ops_center_stack.sh"),
|
||||
)
|
||||
|
||||
def test_build_bash_command_joins_non_empty_args(self) -> None:
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
|
||||
build_bash_command(
|
||||
"drive_ops_center.sh",
|
||||
"driver-resolve",
|
||||
"http://127.0.0.1:8100",
|
||||
"handover_first_gap",
|
||||
),
|
||||
)
|
||||
|
||||
def test_build_bash_command_skips_blank_args(self) -> None:
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad",
|
||||
build_bash_command(
|
||||
"drive_release_hub.sh",
|
||||
"http://127.0.0.1:8100",
|
||||
"",
|
||||
"launchpad",
|
||||
" ",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
391
domain-api/tests/test_ops_job_service.py
Normal file
391
domain-api/tests/test_ops_job_service.py
Normal file
@@ -0,0 +1,391 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_job_service import (
|
||||
_execute_control_plane_job,
|
||||
_serialize_job_row,
|
||||
create_ops_job,
|
||||
dispatch_ops_job,
|
||||
upsert_managed_node,
|
||||
)
|
||||
|
||||
|
||||
class OpsJobServiceTests(unittest.TestCase):
|
||||
def test_serialize_job_row_adds_execution_mode_label(self) -> None:
|
||||
row = (
|
||||
9,
|
||||
"ops-2026041801-abcd12",
|
||||
"logs.collect",
|
||||
"node",
|
||||
"mainland-worker-01",
|
||||
"queued",
|
||||
"remote-agent",
|
||||
"api",
|
||||
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
None,
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
"low",
|
||||
False,
|
||||
"approved",
|
||||
"",
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
None,
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
|
||||
job = _serialize_job_row(row)
|
||||
|
||||
self.assertEqual("remote-agent", job["execution_mode"])
|
||||
self.assertEqual("远端 Agent", job["execution_mode_label"])
|
||||
self.assertEqual("排队中", job["status_label"])
|
||||
self.assertEqual("已审批", job["approval_status_label"])
|
||||
self.assertEqual(["mainland-worker-01"], job["target_node_codes"])
|
||||
self.assertEqual(0, job["steps_total"])
|
||||
self.assertFalse(job["steps_loaded"])
|
||||
self.assertFalse(job["is_compact"])
|
||||
self.assertEqual("ops_job", job["focus_ref"]["kind"])
|
||||
self.assertEqual(9, job["focus_ref"]["job_id"])
|
||||
self.assertIn("目标节点 mainland-worker-01", job["summary_text"])
|
||||
|
||||
def test_serialize_job_row_counts_loaded_steps(self) -> None:
|
||||
row = (
|
||||
10,
|
||||
"ops-2026041801-efgh34",
|
||||
"diagnostics.collect",
|
||||
"node",
|
||||
"mainland-worker-02",
|
||||
"running",
|
||||
"ssh",
|
||||
"web-ui",
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 5, 0),
|
||||
datetime(2026, 4, 18, 10, 5, 2),
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 5, 3),
|
||||
"medium",
|
||||
True,
|
||||
"approved",
|
||||
"admin",
|
||||
datetime(2026, 4, 18, 10, 5, 1),
|
||||
"",
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 5, 2),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
|
||||
job = _serialize_job_row(
|
||||
row,
|
||||
steps=[
|
||||
{"id": 1, "status": "success"},
|
||||
{"id": 2, "status": "running"},
|
||||
{"id": 3, "status": "failed"},
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual("执行中", job["status_label"])
|
||||
self.assertEqual("已审批", job["approval_status_label"])
|
||||
self.assertEqual(3, job["steps_total"])
|
||||
self.assertEqual(1, job["steps_running"])
|
||||
self.assertEqual(1, job["steps_success"])
|
||||
self.assertEqual(1, job["steps_failed"])
|
||||
self.assertEqual(2, job["steps_terminal"])
|
||||
self.assertTrue(job["steps_loaded"])
|
||||
self.assertIn("mainland-worker-02", job["summary_text"])
|
||||
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan")
|
||||
def test_execute_control_plane_bootstrap_uses_managed_node_defaults(
|
||||
self,
|
||||
mock_build_node_agent_bootstrap_plan,
|
||||
mock_list_managed_nodes,
|
||||
) -> None:
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
}
|
||||
]
|
||||
mock_build_node_agent_bootstrap_plan.return_value = (
|
||||
True,
|
||||
"节点 Agent 接入方案已生成",
|
||||
{
|
||||
"token_preview": "agent-...1234",
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = _execute_control_plane_job(
|
||||
"node.bootstrap",
|
||||
target_node_code="mainland-worker-02",
|
||||
payload={
|
||||
"root_dir": "/opt/domaincheck",
|
||||
"expires_in_hours": 12,
|
||||
},
|
||||
requested_by="unit-test",
|
||||
metadata={"source": "test"},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("节点 Agent 接入方案已生成", message)
|
||||
self.assertEqual("control-plane", data["execution_mode"])
|
||||
self.assertEqual("mainland", data["node_region"])
|
||||
self.assertEqual("worker", data["node_role"])
|
||||
mock_build_node_agent_bootstrap_plan.assert_called_once_with(
|
||||
node_code="mainland-worker-02",
|
||||
node_region="mainland",
|
||||
node_role="worker",
|
||||
issued_by="unit-test",
|
||||
expires_in_hours=12,
|
||||
control_plane_base_url="",
|
||||
root_dir="/opt/domaincheck",
|
||||
metadata={
|
||||
"source": "test",
|
||||
"issued_from": "ops-job/control-plane",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"node_region": "mainland",
|
||||
"node_role": "worker",
|
||||
},
|
||||
)
|
||||
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_upsert_managed_node_preserves_remote_fields_when_sync_payload_omits_them(self, mock_get_db) -> None:
|
||||
existing_metadata = {"operator_note": "keep-me"}
|
||||
merged_metadata = {"operator_note": "keep-me", "cluster_status": "online"}
|
||||
existing_row = (
|
||||
"mainland-worker-01",
|
||||
"mainland",
|
||||
"worker",
|
||||
"Mainland Worker 01",
|
||||
"121.204.244.248",
|
||||
22,
|
||||
"root",
|
||||
"key",
|
||||
"stable",
|
||||
False,
|
||||
json.dumps(existing_metadata, ensure_ascii=False),
|
||||
datetime(2026, 4, 18, 1, 0, 0),
|
||||
None,
|
||||
datetime(2026, 4, 18, 1, 5, 0),
|
||||
)
|
||||
returned_row = (
|
||||
"mainland-worker-01",
|
||||
"mainland",
|
||||
"worker",
|
||||
"S244-248",
|
||||
"121.204.244.248",
|
||||
22,
|
||||
"root",
|
||||
"key",
|
||||
"stable",
|
||||
False,
|
||||
json.dumps(merged_metadata, ensure_ascii=False),
|
||||
datetime(2026, 4, 18, 1, 0, 0),
|
||||
None,
|
||||
datetime(2026, 4, 18, 1, 6, 0),
|
||||
)
|
||||
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.side_effect = [existing_row, returned_row]
|
||||
|
||||
ok, message, data = upsert_managed_node(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"title": "S244-248",
|
||||
"metadata": {
|
||||
"cluster_status": "online",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("托管节点已保存", message)
|
||||
node = data["node"]
|
||||
self.assertEqual("121.204.244.248", node["ssh_host"])
|
||||
self.assertEqual("root", node["ssh_user"])
|
||||
self.assertFalse(node["is_enabled"])
|
||||
self.assertEqual("keep-me", node["metadata"]["operator_note"])
|
||||
self.assertEqual("online", node["metadata"]["cluster_status"])
|
||||
|
||||
insert_call = cursor.execute.call_args_list[-1]
|
||||
insert_params = insert_call.args[1]
|
||||
self.assertEqual("121.204.244.248", insert_params[4])
|
||||
self.assertEqual(22, insert_params[5])
|
||||
self.assertEqual("root", insert_params[6])
|
||||
self.assertFalse(insert_params[9])
|
||||
self.assertEqual(merged_metadata, json.loads(insert_params[10]))
|
||||
|
||||
@patch("app.services.ops_agent_service.append_ops_job_event")
|
||||
@patch("app.services.ops_job_service._execute_ssh_job_record")
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_dispatch_ops_job_ssh_executes_via_ssh_record(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_execute_ssh_job_record,
|
||||
mock_append_ops_job_event,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.return_value = (
|
||||
9,
|
||||
"ops-2026041801-abcd12",
|
||||
"logs.collect",
|
||||
"node",
|
||||
"mainland-worker-01",
|
||||
"queued",
|
||||
"ssh",
|
||||
"api",
|
||||
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
None,
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
"low",
|
||||
False,
|
||||
"approved",
|
||||
"",
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
None,
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
mock_execute_ssh_job_record.return_value = (
|
||||
True,
|
||||
"worker logs collected",
|
||||
{"job": {"id": 9, "status": "success"}},
|
||||
)
|
||||
|
||||
ok, message, data = dispatch_ops_job(9)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已按 SSH 执行派发:worker logs collected", message)
|
||||
self.assertEqual("success", data["job"]["status"])
|
||||
mock_execute_ssh_job_record.assert_called_once_with(9)
|
||||
mock_append_ops_job_event.assert_called_once()
|
||||
event_kwargs = mock_append_ops_job_event.call_args.kwargs
|
||||
self.assertEqual("job_dispatch_requested", event_kwargs["event_type"])
|
||||
self.assertEqual("mainland-worker-01", event_kwargs["node_code"])
|
||||
self.assertEqual("ssh", event_kwargs["payload"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.append_ops_job_event")
|
||||
@patch("app.services.ops_job_service.get_ops_job")
|
||||
@patch("app.services.ops_job_service._execute_local_job")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_create_ops_job_auto_uses_local_runtime_for_local_structured_action(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_execute_local_job,
|
||||
mock_get_ops_job,
|
||||
mock_append_ops_job_event,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.return_value = (42,)
|
||||
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"risk_level": "low",
|
||||
"approval_required": False,
|
||||
"blocked": False,
|
||||
"blocking_reasons": [],
|
||||
}
|
||||
mock_execute_local_job.return_value = (
|
||||
True,
|
||||
"worker logs collected",
|
||||
{"executor": "local-runtime"},
|
||||
)
|
||||
mock_get_ops_job.return_value = {
|
||||
"id": 42,
|
||||
"status": "success",
|
||||
"execution_mode": "local-runtime",
|
||||
}
|
||||
|
||||
ok, message, data = create_ops_job(
|
||||
{
|
||||
"action": "logs.collect",
|
||||
"target_node_code": settings.node_code,
|
||||
"payload": {
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
"requested_by": "unit-test",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("worker logs collected", message)
|
||||
self.assertTrue(data["executed_immediately"])
|
||||
self.assertEqual("local-runtime", data["job"]["execution_mode"])
|
||||
mock_execute_local_job.assert_called_once_with(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
)
|
||||
|
||||
insert_call = next(
|
||||
call
|
||||
for call in cursor.execute.call_args_list
|
||||
if "INSERT INTO ops_jobs" in str(call.args[0])
|
||||
)
|
||||
insert_params = insert_call.args[1]
|
||||
self.assertEqual("local-runtime", insert_params[5])
|
||||
mock_append_ops_job_event.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
220
domain-api/tests/test_ops_playbook_service.py
Normal file
220
domain-api/tests/test_ops_playbook_service.py
Normal file
@@ -0,0 +1,220 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_playbook_service import (
|
||||
get_recent_ops_playbook_runs,
|
||||
list_ops_playbook_run_events,
|
||||
preview_ops_playbook,
|
||||
)
|
||||
|
||||
|
||||
class OpsPlaybookServiceTests(unittest.TestCase):
|
||||
def test_onboarding_bootstrap_preview_uses_control_plane(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "onboarding.bootstrap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("playbook 预览生成成功", message)
|
||||
self.assertEqual("onboarding.bootstrap", data["playbook"]["key"])
|
||||
self.assertEqual("control-plane", data["execution_mode"])
|
||||
self.assertEqual("控制面", data["execution_mode_label"])
|
||||
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
|
||||
self.assertEqual("控制面", data["playbook"]["default_execution_mode_label"])
|
||||
self.assertEqual("控制面", data["playbook"]["execution_mode_options"][0]["label"])
|
||||
self.assertTrue(data["auto_approve"])
|
||||
self.assertEqual(1, data["step_count"])
|
||||
step = data["steps"][0]
|
||||
self.assertEqual("bootstrap", step["step_key"])
|
||||
self.assertEqual("node.bootstrap", step["template_key"])
|
||||
self.assertEqual("node.bootstrap", step["action"])
|
||||
self.assertEqual("control-plane", step["execution_mode"])
|
||||
self.assertEqual(1, step["expected_jobs"])
|
||||
|
||||
def test_scene_logs_key_preview_allows_ssh_execution_mode(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "scene.logs.key",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"execution_mode": "ssh",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("playbook 预览生成成功", message)
|
||||
self.assertEqual("ssh", data["execution_mode"])
|
||||
self.assertEqual("SSH", data["execution_mode_label"])
|
||||
self.assertEqual(["remote-agent", "ssh"], data["playbook"]["execution_modes"])
|
||||
self.assertEqual("ssh", data["steps"][0]["execution_mode"])
|
||||
self.assertEqual("SSH", data["steps"][0]["execution_mode_label"])
|
||||
|
||||
def test_onboarding_bootstrap_preview_rejects_unsupported_execution_mode(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "onboarding.bootstrap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"execution_mode": "ssh",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("该 playbook 仅支持 control-plane", message)
|
||||
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
|
||||
|
||||
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_playbook_service.list_ops_jobs")
|
||||
def test_recent_onboarding_bootstrap_run_focus_summary_reflects_agent_state(
|
||||
self,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 31,
|
||||
"job_code": "ops-bootstrap-31",
|
||||
"action": "node.bootstrap",
|
||||
"status": "success",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"created_at": "2026-04-17 10:00:00",
|
||||
"updated_at": "2026-04-17 10:01:00",
|
||||
"metadata": {
|
||||
"playbook": {
|
||||
"run_code": "pbr-bootstrap-1",
|
||||
"key": "onboarding.bootstrap",
|
||||
"title": "生成节点接入工单",
|
||||
"group_key": "onboarding",
|
||||
"group_title": "接管验收",
|
||||
"requested_by": "tester",
|
||||
"execution_mode": "control-plane",
|
||||
"step_key": "bootstrap",
|
||||
"step_title": "生成 Node Agent 接入工单",
|
||||
"step_order": 1,
|
||||
"step_count": 1,
|
||||
"expected_jobs_total": 1,
|
||||
"target_nodes_total": 1,
|
||||
"template_key": "node.bootstrap",
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"is_managed": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_recent_ops_playbook_runs(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["runs"]))
|
||||
run = payload["runs"][0]
|
||||
self.assertEqual("success", run["status"])
|
||||
self.assertEqual("成功", run["status_label"])
|
||||
self.assertEqual("控制面", run["execution_mode_label"])
|
||||
self.assertEqual("warning", run["focus_level"])
|
||||
self.assertIn("待执行 bootstrap 脚本", run["focus_summary"])
|
||||
self.assertEqual(run["focus_summary"], run["summary_text"])
|
||||
self.assertEqual(1, run["steps_total"])
|
||||
self.assertEqual("pbr-bootstrap-1", run["focus_ref"]["run_code"])
|
||||
self.assertEqual("生成 Node Agent 接入工单", run["steps"][0]["title"])
|
||||
self.assertEqual("成功", run["steps"][0]["status_label"])
|
||||
self.assertTrue(run["steps"][0]["summary_text"])
|
||||
|
||||
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_playbook_service.list_ops_job_events_for_jobs")
|
||||
@patch("app.services.ops_playbook_service.list_ops_jobs")
|
||||
def test_playbook_run_events_add_contract_aliases(
|
||||
self,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_ops_job_events_for_jobs,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 41,
|
||||
"job_code": "ops-inspection-41",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"created_at": "2026-04-18 06:30:00",
|
||||
"updated_at": "2026-04-18 06:31:00",
|
||||
"metadata": {
|
||||
"playbook": {
|
||||
"run_code": "pbr-inspection-1",
|
||||
"key": "inspection.standard",
|
||||
"title": "标准巡检",
|
||||
"group_key": "diagnostics",
|
||||
"group_title": "标准巡检",
|
||||
"requested_by": "tester",
|
||||
"execution_mode": "remote-agent",
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"step_order": 2,
|
||||
"step_count": 3,
|
||||
"expected_jobs_total": 3,
|
||||
"target_nodes_total": 1,
|
||||
"template_key": "logs.collect",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_ops_job_events_for_jobs.return_value = [
|
||||
{
|
||||
"id": 501,
|
||||
"event_key": "job-event:501",
|
||||
"job_id": 41,
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "job_running",
|
||||
"level": "info",
|
||||
"level_label": "信息",
|
||||
"message": "开始收集 Worker 日志",
|
||||
"summary": "开始收集 Worker 日志",
|
||||
"summary_text": "远端 Agent 已开始收集 Worker 日志",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 41,
|
||||
"job_code": "ops-inspection-41",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"event_key": "job-event:501",
|
||||
"step_key": "worker_logs",
|
||||
},
|
||||
"payload": {},
|
||||
"created_at": "2026-04-18 06:31:00",
|
||||
"occurred_at": "2026-04-18 06:31:00",
|
||||
"ui_intent": {"kind": "job_detail", "job_id": 41, "focus_event_id": 501},
|
||||
}
|
||||
]
|
||||
|
||||
payload = list_ops_playbook_run_events("pbr-inspection-1", limit=10)
|
||||
|
||||
self.assertEqual("inspection.standard", payload["playbook_run"]["playbook_key"])
|
||||
self.assertEqual(1, len(payload["events"]))
|
||||
event = payload["events"][0]
|
||||
self.assertEqual("收口中", event["job_status_label"])
|
||||
self.assertEqual("远端 Agent 已开始收集 Worker 日志", event["summary_text"])
|
||||
self.assertEqual("worker_logs", event["focus_ref"]["focus_step_key"])
|
||||
self.assertEqual("job-event:501", event["focus_ref"]["event_key"])
|
||||
self.assertEqual("ops_job", event["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:501", event["source_focus_ref"]["event_key"])
|
||||
summary = payload["summary"]
|
||||
self.assertEqual(1, summary["returned_total"])
|
||||
self.assertEqual({"job_running": 1}, summary["event_type_counts"])
|
||||
self.assertEqual({"running": 1}, summary["job_status_counts"])
|
||||
self.assertEqual(10, summary["filters"]["limit"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
253
domain-api/tests/test_ops_policy_service.py
Normal file
253
domain-api/tests/test_ops_policy_service.py
Normal file
@@ -0,0 +1,253 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_policy_service import preview_ops_job_policy
|
||||
|
||||
|
||||
class OpsPolicyServiceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_node_bootstrap_preview_requires_control_plane(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "node.bootstrap",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "remote-agent",
|
||||
"payload": {},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual("critical", preview["risk_level"])
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertTrue(preview["approval_required"])
|
||||
self.assertIn("node.bootstrap 仅支持 control-plane", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_batch_preview_summarizes_node_risks(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
"current_load": 3,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
},
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "batch",
|
||||
"target_node_codes": ["mainland-controller-01", "mainland-worker-01"],
|
||||
"execution_mode": "remote-agent",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual("batch", preview["target_type"])
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertTrue(preview["approval_required"])
|
||||
self.assertEqual(2, preview["target_summary"]["nodes_total"])
|
||||
self.assertEqual(1, preview["target_summary"]["blocked_nodes"])
|
||||
self.assertEqual(1, preview["target_summary"]["approval_nodes"])
|
||||
self.assertEqual(1, preview["target_summary"]["warning_nodes"])
|
||||
self.assertEqual(2, len(preview["target_nodes"]))
|
||||
controller_row = next(item for item in preview["target_nodes"] if item["node_code"] == "mainland-controller-01")
|
||||
self.assertTrue(controller_row["blocked"])
|
||||
self.assertTrue(controller_row["approval_required"])
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_preview_blocks_missing_artifact_and_wrong_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "local-runtime",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("deploy.release 缺少 artifact_url", " ".join(preview["blocking_reasons"]))
|
||||
self.assertIn("deploy.release 仅支持 remote-agent 或 ssh", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_preview_allows_ssh_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "ssh",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(any("仅支持 remote-agent 或 ssh" in item for item in preview["blocking_reasons"]))
|
||||
self.assertTrue(preview["approval_required"])
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_service_status_preview_blocks_control_plane_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "service.status",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "control-plane",
|
||||
"payload": {"service_name": "domaincheck-worker"},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("service.status 当前不支持 control-plane 执行方式", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_logs_collect_preview_blocks_local_runtime_on_remote_node(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "logs.collect",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "local-runtime",
|
||||
"payload": {
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("local-runtime 仅支持当前控制面本机节点", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
89
domain-api/tests/test_ops_release_executor_core.py
Normal file
89
domain-api/tests/test_ops_release_executor_core.py
Normal file
@@ -0,0 +1,89 @@
|
||||
import io
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
|
||||
|
||||
|
||||
class _BytesResponse:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self._data = data
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self, *args, **kwargs) -> bytes:
|
||||
return self._data
|
||||
|
||||
|
||||
def _build_release_archive() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
|
||||
content = b"hello-release"
|
||||
info = tarfile.TarInfo(name="bundle/README.txt")
|
||||
info.size = len(content)
|
||||
archive.addfile(info, io.BytesIO(content))
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
def test_execute_release_action_deploys_archive_and_switches_current(self) -> None:
|
||||
archive_bytes = _build_release_archive()
|
||||
commands: list[tuple[str, ...]] = []
|
||||
|
||||
def run_command(command: list[str], *, timeout: int = 60):
|
||||
commands.append(tuple(command))
|
||||
if command[:2] == ["systemctl", "restart"]:
|
||||
return 0, "", ""
|
||||
if command[:2] == ["systemctl", "is-active"]:
|
||||
return 0, "active", ""
|
||||
return 0, "", ""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ok, message, data = execute_release_action(
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
"install_root": tmpdir,
|
||||
"restart_services": ["domaincheck-worker"],
|
||||
"health_check_services": ["domaincheck-worker"],
|
||||
"health_check_urls": [],
|
||||
"rollback_on_failure": "true",
|
||||
"switch_current": "true",
|
||||
},
|
||||
run_command=run_command,
|
||||
default_api_service_name="domaincheck-api",
|
||||
urlopen_func=lambda request, timeout=120: _BytesResponse(archive_bytes),
|
||||
user_agent="unit-test/0.1",
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("release deployed", message)
|
||||
self.assertEqual("2026.04.18-rc1", data["release_version"])
|
||||
current_link = Path(tmpdir) / "current"
|
||||
self.assertTrue(current_link.is_symlink())
|
||||
release_dir = Path(data["release_dir"])
|
||||
self.assertTrue(release_dir.exists())
|
||||
self.assertTrue((release_dir / ".release-meta.json").exists())
|
||||
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
|
||||
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
|
||||
|
||||
def test_build_remote_release_action_script_is_valid_python(self) -> None:
|
||||
script = build_remote_release_action_script(
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
}
|
||||
)
|
||||
|
||||
compile(script, "<remote-release-script>", "exec")
|
||||
self.assertIn("execute_release_action(", script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
93
domain-api/tests/test_ops_release_service_contracts.py
Normal file
93
domain-api/tests/test_ops_release_service_contracts.py
Normal file
@@ -0,0 +1,93 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.ops_release_service import _serialize_release_row, _serialize_rollout_row
|
||||
|
||||
|
||||
class OpsReleaseServiceContractTests(unittest.TestCase):
|
||||
def test_serialize_release_row_adds_contract_aliases(self) -> None:
|
||||
row = (
|
||||
9,
|
||||
"2026.04.18+bd6bcb2",
|
||||
"stable",
|
||||
"bd6bcb2",
|
||||
"ready",
|
||||
"https://example.com/domaincheck.tar.gz",
|
||||
"sha256:test",
|
||||
"release notes",
|
||||
json.dumps(
|
||||
{
|
||||
"source": "package",
|
||||
"manifest": {
|
||||
"artifact_kind": "tarball",
|
||||
"package_name": "domaincheck-2026.04.18+bd6bcb2.tgz",
|
||||
"included_services": ["domaincheck-api", "domaincheck-worker"],
|
||||
"rollback_hint": "切回上一个 current",
|
||||
},
|
||||
}
|
||||
),
|
||||
"www",
|
||||
datetime(2026, 4, 18, 9, 0, 0),
|
||||
datetime(2026, 4, 18, 9, 5, 0),
|
||||
datetime(2026, 4, 18, 9, 6, 0),
|
||||
)
|
||||
|
||||
payload = _serialize_release_row(row)
|
||||
|
||||
self.assertEqual("已就绪", payload["status_label"])
|
||||
self.assertIn("已就绪", payload["summary_text"])
|
||||
self.assertEqual("release_hub", payload["focus_ref"]["kind"])
|
||||
self.assertEqual(9, payload["focus_ref"]["release_id"])
|
||||
self.assertEqual("release_detail", payload["focus_ref"]["section"])
|
||||
self.assertEqual("tarball", payload["manifest"]["artifact_kind"])
|
||||
self.assertEqual("domaincheck-2026.04.18+bd6bcb2.tgz", payload["manifest"]["package_name"])
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], payload["manifest"]["included_services"])
|
||||
|
||||
def test_serialize_rollout_row_adds_contract_aliases(self) -> None:
|
||||
row = (
|
||||
11,
|
||||
9,
|
||||
"rollout-20260418090000-a1b2",
|
||||
json.dumps({"region": "mainland", "role": "worker"}),
|
||||
json.dumps(
|
||||
{
|
||||
"items": [
|
||||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
|
||||
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
|
||||
]
|
||||
}
|
||||
),
|
||||
json.dumps({"execution_mode": "ssh", "batch_size": 1}),
|
||||
"running",
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
json.dumps(
|
||||
{
|
||||
"status": "running",
|
||||
"launched_targets": 1,
|
||||
"remaining_targets": 1,
|
||||
"active_jobs": 1,
|
||||
"current_batch_index": 1,
|
||||
"batches_total": 2,
|
||||
}
|
||||
),
|
||||
"web-ui",
|
||||
datetime(2026, 4, 18, 9, 10, 0),
|
||||
datetime(2026, 4, 18, 9, 11, 0),
|
||||
)
|
||||
|
||||
payload = _serialize_rollout_row(row)
|
||||
|
||||
self.assertEqual("执行中", payload["status_label"])
|
||||
self.assertEqual(["mainland-worker-01", "mainland-worker-02"], payload["target_node_codes"])
|
||||
self.assertIn("当前已覆盖 1/2 台节点", payload["summary_text"])
|
||||
self.assertEqual("release_rollout", payload["focus_ref"]["kind"])
|
||||
self.assertEqual(11, payload["focus_ref"]["rollout_id"])
|
||||
self.assertEqual("rollout_jobs", payload["focus_ref"]["section"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
432
domain-api/tests/test_ops_release_service_execution_modes.py
Normal file
432
domain-api/tests/test_ops_release_service_execution_modes.py
Normal file
@@ -0,0 +1,432 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
build_release_execution_mode_recommendation,
|
||||
build_release_rollout_gate,
|
||||
build_rollout_target_operational_readiness,
|
||||
)
|
||||
|
||||
|
||||
class _EmptyCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, query, params=None) -> None:
|
||||
self.query = query
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
|
||||
class _EmptyConnection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return _EmptyCursor()
|
||||
|
||||
|
||||
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
|
||||
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
|
||||
"recovery_decision": {
|
||||
"action": "run_acceptance",
|
||||
"label": "执行接管验收",
|
||||
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
|
||||
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
|
||||
"window": "acceptance",
|
||||
},
|
||||
}
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="ssh",
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", readiness["execution_mode"])
|
||||
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
|
||||
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
|
||||
self.assertEqual([], readiness["blocking_reasons"])
|
||||
self.assertTrue(readiness["rows"][0]["ssh_ready"])
|
||||
self.assertTrue(readiness["rows"][0]["execution_ready"])
|
||||
self.assertTrue(readiness["rows"][0]["agent_managed"])
|
||||
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
|
||||
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
|
||||
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
|
||||
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
|
||||
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
|
||||
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
|
||||
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||||
"summary": "当前节点无需额外恢复动作。",
|
||||
"recovery_decision": {
|
||||
"action": "noop",
|
||||
"label": "当前无需额外恢复动作",
|
||||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||||
"command_hint": "",
|
||||
"window": "none",
|
||||
},
|
||||
}
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_user": "root",
|
||||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||||
"last_seen_at": "2026-04-18 08:00:00",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
|
||||
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
|
||||
self.assertEqual("release_hub", row["focus_ref"]["kind"])
|
||||
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
|
||||
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
|
||||
self.assertEqual("noop", row["recovery_action"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.249",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
}
|
||||
]
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
|
||||
"summary": "已签发有效 Agent Token,但节点尚未完成 register/heartbeat。",
|
||||
"recovery_decision": {
|
||||
"action": "bootstrap_run",
|
||||
"label": "签发接入工单",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
|
||||
"window": "bootstrap",
|
||||
},
|
||||
}
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("bootstrap_run", row["recovery_action"])
|
||||
self.assertEqual("bootstrap_run", row["recommended_action_code"])
|
||||
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
|
||||
self.assertTrue(
|
||||
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||||
"summary": "当前节点无需额外恢复动作。",
|
||||
"recovery_decision": {
|
||||
"action": "noop",
|
||||
"label": "当前无需额外恢复动作",
|
||||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||||
"command_hint": "",
|
||||
"window": "none",
|
||||
},
|
||||
}
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-03",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-03",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.250",
|
||||
"ssh_user": "root",
|
||||
"metadata": {
|
||||
"delivery_queue": {
|
||||
"state": "dead_letter",
|
||||
"label": "死信 1",
|
||||
"reason": "当前存在 1 条死信记录",
|
||||
"pending_count": 0,
|
||||
"dead_letter_count": 1,
|
||||
"oldest_dead_letter_at": "2026-04-18 09:01:00",
|
||||
}
|
||||
},
|
||||
"last_seen_at": "2026-04-18 09:05:00",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("dead_letter", row["delivery_queue_state"])
|
||||
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
|
||||
self.assertTrue(
|
||||
any("死信" in item for item in row["blocking_reasons"])
|
||||
)
|
||||
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
|
||||
self.assertTrue(
|
||||
any("死信回执" in item for item in readiness["blocking_reasons"])
|
||||
)
|
||||
self.assertTrue(
|
||||
any("死信队列" in item for item in readiness["recommendations"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||||
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
|
||||
mock_build_readiness.return_value = {
|
||||
"execution_mode": "ssh",
|
||||
"summary": {
|
||||
"nodes_total": 2,
|
||||
"execution_ready_nodes": 1,
|
||||
"inspection_healthy_nodes": 2,
|
||||
},
|
||||
"rows": [],
|
||||
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口,当前不适合直接 SSH 发布。"],
|
||||
"warning_reasons": [],
|
||||
"recommendations": [],
|
||||
}
|
||||
|
||||
gate = build_release_rollout_gate(
|
||||
{
|
||||
"id": 11,
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
execution_mode="ssh",
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", gate["execution_mode"])
|
||||
self.assertEqual("SSH", gate["execution_mode_label"])
|
||||
self.assertEqual("blocked", gate["status"])
|
||||
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
|
||||
|
||||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||||
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
|
||||
mock_build_readiness.return_value = {
|
||||
"execution_mode": "remote-agent",
|
||||
"summary": {
|
||||
"nodes_total": 2,
|
||||
"execution_ready_nodes": 1,
|
||||
"inspection_healthy_nodes": 2,
|
||||
},
|
||||
"rows": [],
|
||||
"blocking_reasons": ["有 1 台目标节点 Agent 未在线,当前不适合直接 remote-agent 发布。"],
|
||||
"warning_reasons": [],
|
||||
"recommendations": [],
|
||||
}
|
||||
|
||||
gate = build_release_rollout_gate(
|
||||
{
|
||||
"id": 12,
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
self.assertEqual("remote-agent", gate["execution_mode"])
|
||||
self.assertEqual("远端 Agent", gate["execution_mode_label"])
|
||||
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
|
||||
|
||||
@patch("app.services.ops_release_service.build_release_rollout_gate")
|
||||
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
|
||||
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
|
||||
if execution_mode == "ssh":
|
||||
return {
|
||||
"status": "ready",
|
||||
"status_label": "可发 Rollout",
|
||||
"summary": "SSH 路径已收口",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"blocking_reasons": [],
|
||||
"warning_reasons": [],
|
||||
"operational_readiness": {
|
||||
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
|
||||
"rows": [],
|
||||
},
|
||||
}
|
||||
return {
|
||||
"status": "blocked",
|
||||
"status_label": "存在阻断",
|
||||
"summary": "remote-agent 尚未收口",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"blocking_reasons": ["Agent 未就绪"],
|
||||
"warning_reasons": [],
|
||||
"operational_readiness": {
|
||||
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
|
||||
"rows": [],
|
||||
},
|
||||
}
|
||||
|
||||
mock_build_gate.side_effect = _fake_gate
|
||||
|
||||
recommendation = build_release_execution_mode_recommendation(
|
||||
{
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18-rc2",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", recommendation["recommended_mode"])
|
||||
self.assertEqual("SSH", recommendation["recommended_mode_label"])
|
||||
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
|
||||
self.assertIn("SSH", recommendation["reason"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
212
domain-api/tests/test_ops_release_service_launchpad.py
Normal file
212
domain-api/tests/test_ops_release_service_launchpad.py
Normal file
@@ -0,0 +1,212 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
_build_release_launchpad_status,
|
||||
_compact_latest_smart_rollout_preview,
|
||||
)
|
||||
|
||||
|
||||
class OpsReleaseServiceLaunchpadTests(unittest.TestCase):
|
||||
def test_launchpad_status_blocks_when_package_missing(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": False},
|
||||
latest_package_preview={},
|
||||
worker_preview={},
|
||||
control_preview={},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("release_package", status["recommended_action_code"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
self.assertEqual("release_hub", status["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
|
||||
|
||||
def test_launchpad_status_blocks_when_rollout_preview_has_blockers(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"policy_preview": {"blocked": True}},
|
||||
control_preview={"policy_preview": {"blocked": False}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("fix_rollout_blockers", status["recommended_action_code"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_blocks_when_final_signoff_missing(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={
|
||||
"available": True,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": False,
|
||||
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("release_prepare", status["recommended_action_code"])
|
||||
self.assertIn("final_release_report does not match latest_release: package_name", status["summary"])
|
||||
self.assertIn("release-prepare", status["recommended_command"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_prefers_runtime_refresh_when_runtime_schema_is_stale(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True, "final_release_ok": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
runtime_build_info={
|
||||
"runtime_schema_stale": True,
|
||||
"route_surface_complete": True,
|
||||
"route_surface_missing_keys": ["ops_node_handover_bootstrap_plan"],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("api-restart", status["recommended_action_code"])
|
||||
self.assertEqual("待刷新运行时", status["status_label"])
|
||||
self.assertTrue(status["runtime_refresh_required"])
|
||||
self.assertTrue(status["runtime_schema_stale"])
|
||||
self.assertIn("runtime-refresh-recover", status["recommended_command"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_marks_attention_when_confirmation_required(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"requires_confirmation": True, "policy_preview": {}},
|
||||
control_preview={"policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("review_smart_rollout_preview", status["recommended_action_code"])
|
||||
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
|
||||
|
||||
def test_launchpad_status_ready_when_worker_rollout_available(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True, "execution_mode": "ssh", "execution_mode_label": "SSH"}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("ready", status["status"])
|
||||
self.assertEqual("publish_latest_worker", status["recommended_action_code"])
|
||||
self.assertEqual("ssh", status["recommended_execution_mode"])
|
||||
self.assertEqual("SSH", status["recommended_execution_mode_label"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_requires_managed_nodes_when_no_targets_available(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("fix_managed_nodes", status["recommended_action_code"])
|
||||
self.assertEqual("release_hub", status["focus_ref"]["kind"])
|
||||
|
||||
def test_launchpad_status_prefers_bootstrap_recovery_when_gap_row_has_recovery_action(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={
|
||||
"smart_rollout": {"available": False},
|
||||
"policy_preview": {
|
||||
"blocked": True,
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_label": "签发接入工单",
|
||||
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("bootstrap_run", status["recommended_action_code"])
|
||||
self.assertEqual("mainland-worker-02", status["recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", status["recommended_recovery_label"])
|
||||
self.assertEqual("当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。", status["recommended_recovery_summary"])
|
||||
self.assertEqual(1, status["onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, status["onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(1, status["onboarding_gap_nodes_total"])
|
||||
self.assertIn("node-bootstrap-run", status["recommended_command"])
|
||||
|
||||
def test_compact_latest_smart_rollout_preview_adds_rollout_preview_alias(self) -> None:
|
||||
preview = _compact_latest_smart_rollout_preview(
|
||||
"智能 Rollout 预检通过",
|
||||
{
|
||||
"release": {
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18+bd6bcb2",
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
"package": {
|
||||
"available": True,
|
||||
"package_name": "domaincheck-2026.04.18+bd6bcb2",
|
||||
"package_type": "tar.gz",
|
||||
"sha256": "sha256:test",
|
||||
"metadata": {
|
||||
"manifest": {
|
||||
"included_services": ["domaincheck-api", "domaincheck-worker"],
|
||||
}
|
||||
},
|
||||
},
|
||||
"smart_rollout": {
|
||||
"available": True,
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"summary": "已按 Worker 预选 2 台节点。",
|
||||
"target_selector": {"node_codes": ["mainland-worker-01", "mainland-worker-02"]},
|
||||
"target_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"delivery_queue_state": "healthy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"delivery_queue_state": "retrying",
|
||||
},
|
||||
],
|
||||
},
|
||||
"preview": {
|
||||
"target_summary": {"nodes_total": 2},
|
||||
"batch_plan": {"batches_total": 2, "first_batch_size": 1},
|
||||
"release_gate": {"status": "ready", "summary": "当前可推进。"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("remote-agent", preview["rollout_preview"]["execution_mode"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["target_nodes_total"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["expected_batches_total"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["expected_jobs_total"])
|
||||
self.assertEqual("release_hub", preview["rollout_preview"]["focus_ref"]["kind"])
|
||||
self.assertEqual("healthy", preview["smart_rollout"]["target_nodes"][0]["delivery_queue_state"])
|
||||
self.assertEqual("retrying", preview["smart_rollout"]["target_nodes"][1]["delivery_queue_state"])
|
||||
self.assertEqual("rollout_preview", preview["rollout_preview"]["focus_ref"]["section"])
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], preview["package"]["manifest"]["included_services"])
|
||||
915
domain-api/tests/test_ops_release_service_package_metadata.py
Normal file
915
domain-api/tests/test_ops_release_service_package_metadata.py
Normal file
@@ -0,0 +1,915 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
build_smart_release_rollout_plan,
|
||||
create_release_and_smart_rollout_from_latest_package,
|
||||
create_release_from_latest_package,
|
||||
create_smart_release_rollout,
|
||||
get_latest_release_package_metadata,
|
||||
get_release_package_archive_path,
|
||||
get_release_package_sha256_path,
|
||||
)
|
||||
|
||||
|
||||
class OpsReleaseServicePackageMetadataTests(unittest.TestCase):
|
||||
def test_get_latest_release_package_metadata_reads_latest_linux_package(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_20260418_120000"
|
||||
staging_path = release_root / package_name
|
||||
archive_path = release_root / f"{package_name}.tar.gz"
|
||||
sha256_path = release_root / f"{package_name}.sha256.txt"
|
||||
|
||||
staging_path.mkdir(parents=True, exist_ok=True)
|
||||
archive_path.write_bytes(b"linux-package")
|
||||
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
|
||||
(staging_path / "release_manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"included": {
|
||||
"docs": ["08_domainCheck_交付说明.md"],
|
||||
"scripts": ["package_domain_release.sh", "verify_domain_release.sh"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"generated_at": "2026-04-18T12:00:00",
|
||||
"staging_path": str(staging_path),
|
||||
"archive_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "test-sha256",
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": package_name,
|
||||
"archive_path": str(archive_path),
|
||||
"sha256": "test-sha256",
|
||||
},
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertTrue(data["available"])
|
||||
self.assertEqual(package_name, data["package_name"])
|
||||
self.assertEqual("tar.gz", data["package_type"])
|
||||
self.assertTrue(data["archive_present"])
|
||||
self.assertTrue(data["sha256_present"])
|
||||
self.assertTrue(data["staging_present"])
|
||||
self.assertEqual("test-sha256", data["sha256"])
|
||||
self.assertEqual(f"/api/v1/ops/releases/packages/{package_name}/download", data["download_path"])
|
||||
self.assertEqual(package_name, data["release_version_suggestion"])
|
||||
self.assertIn("自检通过", data["notes_suggestion"])
|
||||
self.assertTrue(data["final_release_ok"])
|
||||
self.assertTrue(data["final_release_report_available"])
|
||||
self.assertTrue(data["final_release_report_matches_latest"])
|
||||
self.assertEqual("ready", data["final_release_gate_decision"])
|
||||
self.assertEqual([], data["final_release_gate_blocked_reasons"])
|
||||
self.assertEqual(package_name, data["metadata"]["manifest"]["package_name"])
|
||||
|
||||
def test_get_release_package_paths_support_metadata_fallback_for_zip(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_windows_demo"
|
||||
package_dir = release_root / "exports"
|
||||
package_dir.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = package_dir / f"{package_name}.zip"
|
||||
sha256_path = package_dir / f"{package_name}.sha256.txt"
|
||||
|
||||
archive_path.write_bytes(b"zip-package")
|
||||
sha256_path.write_text("sha256=zip-sha256", encoding="utf-8")
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "zip",
|
||||
"generated_at": "2026-04-18T12:10:00",
|
||||
"zip_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "zip-sha256",
|
||||
"smoke_test_ok": False,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
resolved_archive = get_release_package_archive_path(package_name)
|
||||
resolved_sha256 = get_release_package_sha256_path(package_name)
|
||||
metadata = get_latest_release_package_metadata()
|
||||
|
||||
self.assertEqual(archive_path, resolved_archive)
|
||||
self.assertEqual(sha256_path, resolved_sha256)
|
||||
self.assertEqual("zip", metadata["package_type"])
|
||||
self.assertEqual(archive_path.name, metadata["archive_filename"])
|
||||
self.assertFalse(metadata["smoke_test_ok"])
|
||||
|
||||
def test_get_latest_release_package_metadata_returns_unavailable_when_missing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": tmpdir}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertFalse(data["available"])
|
||||
self.assertIn("latest_release.json", data["reason"])
|
||||
|
||||
def test_get_latest_release_package_metadata_marks_stale_final_release_report(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_20260418_130000"
|
||||
archive_path = release_root / f"{package_name}.tar.gz"
|
||||
sha256_path = release_root / f"{package_name}.sha256.txt"
|
||||
|
||||
archive_path.write_bytes(b"linux-package")
|
||||
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"archive_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "test-sha256",
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": "domaincheck_release_20260418_old",
|
||||
"archive_path": str(release_root / "domaincheck_release_20260418_old.tar.gz"),
|
||||
"sha256": "old-sha256",
|
||||
},
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertTrue(data["available"])
|
||||
self.assertFalse(data["final_release_ok"])
|
||||
self.assertTrue(data["final_release_report_available"])
|
||||
self.assertTrue(data["final_release_report_ok"])
|
||||
self.assertFalse(data["final_release_report_matches_latest"])
|
||||
self.assertIn("package_name", data["final_release_report_stale_reason"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_builds_payload(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_120000",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"notes_suggestion": "导入自本机发布包 domaincheck_release_20260418_120000.tar.gz;自检通过",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": True,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": True,
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {}
|
||||
mock_create_release.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {
|
||||
"id": 7,
|
||||
"release_version": "domaincheck_release_20260418_120000",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"created_by": "test-suite",
|
||||
},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已创建", message)
|
||||
mock_create_release.assert_called_once()
|
||||
create_payload = mock_create_release.call_args.args[0]
|
||||
self.assertEqual("domaincheck_release_20260418_120000", create_payload["release_version"])
|
||||
self.assertEqual("bd6bcb2", create_payload["commit_sha"])
|
||||
self.assertEqual(
|
||||
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
create_payload["artifact_url"],
|
||||
)
|
||||
self.assertEqual("test-sha256", create_payload["checksum"])
|
||||
self.assertEqual("latest_release_package", create_payload["metadata"]["created_from"])
|
||||
self.assertEqual(
|
||||
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
create_payload["metadata"]["package_source"]["download_url"],
|
||||
)
|
||||
self.assertFalse(data["deduplicated"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_is_idempotent(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_120000",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": True,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": True,
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
}
|
||||
mock_get_release_by_version.return_value = {
|
||||
"id": 9,
|
||||
"release_version": "domaincheck_release_20260418_120000",
|
||||
}
|
||||
|
||||
ok, message, data = create_release_from_latest_package({}, control_plane_base_url="https://ops.example.com")
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已存在,按幂等返回", message)
|
||||
self.assertTrue(data["deduplicated"])
|
||||
self.assertEqual(9, data["release"]["id"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_blocks_when_final_signoff_missing(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_130000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_130000",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"smoke_test_ok": False,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": False,
|
||||
"final_release_report_ok": None,
|
||||
"final_release_report_matches_latest": None,
|
||||
"final_release_gate_decision": "",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_130000",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {}
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"created_by": "test-suite",
|
||||
},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
self.assertIn("smoke_test_failed_or_skipped", data["final_release_gate"]["blocked_reasons"])
|
||||
self.assertIn("final_release_report_missing", data["final_release_gate"]["blocked_reasons"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_blocks_when_final_report_is_stale(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_130001",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_130001",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": False,
|
||||
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_130001",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {
|
||||
"id": 19,
|
||||
"release_version": "domaincheck_release_20260418_130001",
|
||||
}
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{"created_by": "test-suite"},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
self.assertEqual(
|
||||
"final_release_report does not match latest_release: package_name",
|
||||
data["final_release_gate"]["stale_reason"],
|
||||
)
|
||||
self.assertIn("final_release_report_not_matching_latest", data["final_release_gate"]["blocked_reasons"])
|
||||
self.assertEqual(19, data["release"]["id"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_prefers_remote_ready_workers(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 1,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": True,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
control_plan = build_smart_release_rollout_plan("control")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual("worker", worker_plan["mode"])
|
||||
self.assertEqual(["mainland-worker-01"], worker_plan["target_selector"]["node_codes"])
|
||||
self.assertEqual(["domaincheck-worker"], worker_plan["policy"]["deploy_payload"]["restart_services"])
|
||||
|
||||
self.assertTrue(control_plan["available"])
|
||||
self.assertEqual("control", control_plan["mode"])
|
||||
self.assertEqual(["mainland-controller-01"], control_plan["target_selector"]["node_codes"])
|
||||
self.assertIn("domaincheck-api", control_plan["policy"]["deploy_payload"]["restart_services"])
|
||||
self.assertEqual("remote-agent", worker_plan["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_falls_back_to_ssh_ready_workers(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "busy",
|
||||
"cluster_current_load": 1,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual("worker", worker_plan["mode"])
|
||||
self.assertEqual("ssh", worker_plan["execution_mode"])
|
||||
self.assertEqual("SSH", worker_plan["execution_mode_label"])
|
||||
self.assertEqual(
|
||||
["mainland-worker-01", "mainland-worker-02"],
|
||||
worker_plan["target_selector"]["node_codes"],
|
||||
)
|
||||
self.assertEqual("ssh", worker_plan["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_excludes_dead_letter_nodes_from_ready_candidates(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "dead_letter",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "healthy",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual(["mainland-worker-02"], worker_plan["target_selector"]["node_codes"])
|
||||
self.assertEqual("healthy", worker_plan["target_nodes"][0]["delivery_queue_state"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_returns_unavailable_when_only_dead_letter_nodes_exist(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "dead_letter",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertFalse(worker_plan["available"])
|
||||
self.assertIn("死信回执", worker_plan["reason"])
|
||||
self.assertEqual("dead_letter", worker_plan["target_nodes"][0]["delivery_queue_state"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_stops_when_latest_package_gate_blocks(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
False,
|
||||
"最新发布包尚未完成最终签收,暂不允许创建 Release 或发起基于最新包的智能 Rollout。",
|
||||
{
|
||||
"package": {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_blocked",
|
||||
"final_release_ok": False,
|
||||
},
|
||||
"final_release_gate": {
|
||||
"ready": False,
|
||||
"status": "blocked",
|
||||
"blocked_reasons": ["final_release_report_missing"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker"},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
mock_list_managed_nodes_with_agent_state.assert_not_called()
|
||||
mock_preview_ops_job_policy.assert_not_called()
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_requires_confirmation_before_rollout(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 21, "release_version": "domaincheck_release_20260418_120000"},
|
||||
"deduplicated": False,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": True,
|
||||
"warnings": ["本次 rollout 覆盖 control 节点,建议审批。"],
|
||||
"release_gate": {"status_label": "待补巡检", "summary": "建议先确认控制面健康。"},
|
||||
}
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "control"},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已创建,智能 Rollout 需要确认", message)
|
||||
self.assertTrue(data["requires_confirmation"])
|
||||
self.assertFalse(data["rollout_created"])
|
||||
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_creates_rollout_after_confirmation(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 22, "release_version": "domaincheck_release_20260418_120000"},
|
||||
"deduplicated": True,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 5, "release_id": 22, "rollout_code": "rollout-20260418-demo"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker", "confirm_risky": True},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["rollout_created"])
|
||||
self.assertEqual(5, data["rollout"]["id"])
|
||||
mock_create_release_rollout.assert_called_once()
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
|
||||
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_uses_ssh_when_only_ssh_path_is_ready(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 23, "release_version": "domaincheck_release_20260418_ssh"},
|
||||
"deduplicated": True,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 6, "release_id": 23, "rollout_code": "rollout-20260418-ssh"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker", "confirm_risky": True},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["rollout_created"])
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual("ssh", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_release")
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_create_smart_release_rollout_for_existing_release_requires_confirmation(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
mock_get_release,
|
||||
) -> None:
|
||||
mock_get_release.return_value = {
|
||||
"id": 41,
|
||||
"release_version": "domaincheck_release_existing",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": True,
|
||||
"warnings": ["控制面节点建议审批。"],
|
||||
"release_gate": {"status_label": "待审批", "summary": "需要审批。"},
|
||||
}
|
||||
|
||||
ok, message, data = create_smart_release_rollout(41, {"mode": "control"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("智能 Rollout 需要确认", message)
|
||||
self.assertFalse(data["release_created"])
|
||||
self.assertTrue(data["requires_confirmation"])
|
||||
self.assertFalse(data["rollout_created"])
|
||||
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.get_release")
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_create_smart_release_rollout_for_existing_release_creates_rollout(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
mock_get_release,
|
||||
) -> None:
|
||||
mock_get_release.return_value = {
|
||||
"id": 42,
|
||||
"release_version": "domaincheck_release_existing",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 8, "release_id": 42, "rollout_code": "rollout-20260418-existing"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_smart_release_rollout(42, {"mode": "worker", "confirm_risky": True})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertFalse(data["release_created"])
|
||||
self.assertTrue(data["rollout_created"])
|
||||
self.assertEqual(8, data["rollout"]["id"])
|
||||
mock_create_release_rollout.assert_called_once()
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
|
||||
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
118
domain-api/tests/test_ops_runtime_executor_service.py
Normal file
118
domain-api/tests/test_ops_runtime_executor_service.py
Normal file
@@ -0,0 +1,118 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_runtime_executor_service import execute_local_support_action, execute_ssh_action
|
||||
|
||||
|
||||
class OpsRuntimeExecutorServiceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_ssh_action_parses_structured_json_result(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["ssh"],
|
||||
returncode=0,
|
||||
stdout='login banner\n{"ok": true, "message": "health snapshot collected", "result": {"checks": {"api": {"state": "active"}}}}\n',
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_ssh_action(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_port": 22,
|
||||
"ssh_user": "root",
|
||||
},
|
||||
"health.snapshot",
|
||||
{},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("health snapshot collected", message)
|
||||
self.assertEqual("active", data["checks"]["api"]["state"])
|
||||
self.assertEqual("ssh", data["transport"]["executor"])
|
||||
self.assertEqual("121.204.244.248", data["transport"]["ssh_host"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual("ssh", issued_command[0])
|
||||
self.assertIn("root@121.204.244.248", issued_command)
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_collects_logs_with_payload(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["journalctl"],
|
||||
returncode=0,
|
||||
stdout="log line 1\nlog line 2",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_local_support_action(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 180,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("log line 1\nlog line 2", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual(180, data["lines"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual(
|
||||
["journalctl", "-u", "domaincheck-worker", "-n", "180", "--no-pager"],
|
||||
issued_command,
|
||||
)
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_falls_back_to_worker_log_file(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["journalctl"],
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="No journal files were opened due to insufficient permissions.",
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
log_path = Path(temp_dir) / "detect_worker.log"
|
||||
log_path.write_text("fallback-1\nfallback-2\n", encoding="utf-8")
|
||||
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
|
||||
ok, message, data = execute_local_support_action(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 180,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("fallback-1\nfallback-2", message)
|
||||
self.assertEqual("file_fallback", data["collection_source"])
|
||||
self.assertTrue(data["fallback_used"])
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_restarts_service(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["systemctl"],
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_local_support_action(
|
||||
"service.restart",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("domaincheck-worker restart completed", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual("restart", data["systemctl_action"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual(["systemctl", "restart", "domaincheck-worker"], issued_command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
75
domain-api/tests/test_ops_schema_init.py
Normal file
75
domain-api/tests/test_ops_schema_init.py
Normal file
@@ -0,0 +1,75 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import app.services.ops_agent_service as ops_agent_service
|
||||
import app.services.ops_job_service as ops_job_service
|
||||
|
||||
|
||||
class OpsSchemaInitTests(unittest.TestCase):
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_job_service._OPS_SCHEMA_READY
|
||||
ops_job_service._OPS_SCHEMA_READY = False
|
||||
try:
|
||||
ops_job_service.ensure_ops_schema()
|
||||
ops_job_service.ensure_ops_schema()
|
||||
finally:
|
||||
ops_job_service._OPS_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertEqual(1, mock_get_db.call_count)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(ops_job_service._OPS_SCHEMA_ADVISORY_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_agent_service.ensure_ops_schema")
|
||||
@patch("app.services.ops_agent_service.get_db")
|
||||
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_ensure_ops_schema,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
|
||||
try:
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
finally:
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertEqual(2, mock_ensure_ops_schema.call_count)
|
||||
self.assertEqual(1, mock_get_db.call_count)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(ops_agent_service._OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
607
domain-api/tests/test_ops_service_activity.py
Normal file
607
domain-api/tests/test_ops_service_activity.py
Normal file
@@ -0,0 +1,607 @@
|
||||
import unittest
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_activity_stream
|
||||
|
||||
|
||||
class OpsServiceActivityTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _runtime_status(detect: Optional[dict] = None) -> dict:
|
||||
return {"detect": dict(detect or {})}
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_uses_job_detail_for_bootstrap_jobs(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"agent_state_label": "待接入",
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 51,
|
||||
"job_code": "ops-bootstrap-51",
|
||||
"action": "node.bootstrap",
|
||||
"status": "success",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"created_at": "2026-04-17 10:00:00",
|
||||
"updated_at": "2026-04-17 10:01:00",
|
||||
"result": {
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("节点接入工单", item["title"])
|
||||
self.assertEqual("job_detail", item["ui_intent"]["kind"])
|
||||
self.assertEqual("job_detail", item["ui_intent_kind"])
|
||||
self.assertEqual("控制面", item["execution_mode_label"])
|
||||
self.assertEqual("成功", item["status_label"])
|
||||
self.assertIn("尚未回连 Agent", item["summary"])
|
||||
self.assertEqual(item["summary"], item["summary_text"])
|
||||
self.assertIn("方式 控制面", item["meta_text"])
|
||||
self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"])
|
||||
self.assertEqual(["mainland-worker-02"], item["target_node_codes"])
|
||||
self.assertEqual(51, item["focus_ref"]["job_id"])
|
||||
self.assertEqual("node.bootstrap", item["focus_ref"]["action"])
|
||||
self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_includes_runbook_sequences_after_live_events(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-001",
|
||||
"playbook_title": "标准巡检",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"reason": "当前没有足够的执行面接管覆盖。",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"primary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "fix_managed_nodes",
|
||||
"driver_action_label": "补执行面接管",
|
||||
"driver_node_codes": ["mainland-controller-01"],
|
||||
"message": "当前建议先补执行面接管。",
|
||||
"resolved_at": "2026-04-18 09:12:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertEqual("playbook_run", payload["items"][0]["kind"])
|
||||
self.assertEqual("runbook_sequence", payload["items"][1]["kind"])
|
||||
runbook_item = payload["items"][1]
|
||||
self.assertEqual("release_progression", runbook_item["sequence_key"])
|
||||
self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"])
|
||||
self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"])
|
||||
self.assertEqual("attention", runbook_item["status"])
|
||||
self.assertIn("当前建议动作:补执行面接管", runbook_item["summary"])
|
||||
self.assertIn("release_progression", runbook_item["meta_text"])
|
||||
self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-ssh-001",
|
||||
"playbook_title": "关键现场日志",
|
||||
"group_title": "执行现场",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"steps": [{"status": "running"}],
|
||||
"step_count": 1,
|
||||
"jobs_total": 2,
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 61,
|
||||
"job_code": "ops-ssh-61",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:10:00",
|
||||
"created_at": "2026-04-18 09:09:00",
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertEqual("SSH", payload["items"][0]["execution_mode_label"])
|
||||
self.assertIn("执行 SSH", payload["items"][0]["meta_text"])
|
||||
self.assertEqual("SSH", payload["items"][1]["execution_mode_label"])
|
||||
self.assertIn("方式 SSH", payload["items"][1]["summary"])
|
||||
self.assertIn("执行 SSH", payload["items"][1]["meta_text"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_prefers_job_result_summary_and_focus_ref(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"status": "success",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "codex",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:20:00",
|
||||
"created_at": "2026-04-18 09:19:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"result": {
|
||||
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"event_key": "job-event:888",
|
||||
"step_key": "worker_logs",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", item["summary"])
|
||||
self.assertEqual("ops_job", item["focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:888", item["focus_ref"]["event_key"])
|
||||
self.assertEqual("worker_logs", item["focus_ref"]["step_key"])
|
||||
self.assertIn("执行 远端 Agent", item["meta_text"])
|
||||
self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "codex",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 10:20:00",
|
||||
"created_at": "2026-04-18 10:19:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"result": {
|
||||
"summary_text": "节点端已经收到采集任务。",
|
||||
"start_delivery_state": "failed_local",
|
||||
"start_delivery_error": "temporary offline",
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("attention", item["status"])
|
||||
self.assertEqual("running", item["job_status"])
|
||||
self.assertEqual("failed_local", item["start_delivery_state"])
|
||||
self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"])
|
||||
self.assertIn("开始回执未成功送达控制面", item["summary"])
|
||||
self.assertIn("temporary offline", item["summary"])
|
||||
self.assertIn("开始回执异常", item["meta_text"])
|
||||
self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"])
|
||||
self.assertEqual("attention", payload["summary"]["status"])
|
||||
self.assertEqual("待处理", payload["summary"]["status_label"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_surfaces_execution_mode_label_for_rollout(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = [
|
||||
{
|
||||
"id": 201,
|
||||
"release_id": 18,
|
||||
"rollout_code": "rollout-20260418093000-a1b2",
|
||||
"policy": {
|
||||
"execution_mode": "ssh",
|
||||
},
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"target_nodes": [{"node_code": "mainland-worker-01"}],
|
||||
"jobs_total": 2,
|
||||
"jobs_created": 1,
|
||||
"batch_cursor": 1,
|
||||
"batches_total": 2,
|
||||
"result_summary": {},
|
||||
"summary_text": "当前已覆盖 1/1 台节点,仍有 1 条任务执行中。",
|
||||
"focus_ref": {
|
||||
"kind": "release_rollout",
|
||||
"rollout_id": 201,
|
||||
"rollout_code": "rollout-20260418093000-a1b2",
|
||||
"release_id": 18,
|
||||
"section": "rollout_jobs",
|
||||
},
|
||||
"created_by": "web-ui",
|
||||
"created_at": "2026-04-18 09:30:00",
|
||||
"updated_at": "2026-04-18 09:31:00",
|
||||
}
|
||||
]
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("rollout", item["kind"])
|
||||
self.assertEqual("SSH", item["execution_mode_label"])
|
||||
self.assertIn("执行 SSH", item["meta_text"])
|
||||
self.assertEqual("执行中", item["status_label"])
|
||||
self.assertIn("当前已覆盖 1/1 台节点", item["summary"])
|
||||
self.assertEqual("release_rollout", item["focus_ref"]["kind"])
|
||||
self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_supports_execution_mode_filter(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-ssh-001",
|
||||
"playbook_title": "关键现场日志",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"steps": [{"status": "running"}],
|
||||
"step_count": 1,
|
||||
"jobs_total": 1,
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 61,
|
||||
"job_code": "ops-ssh-61",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:10:00",
|
||||
"created_at": "2026-04-18 09:09:00",
|
||||
},
|
||||
{
|
||||
"id": 62,
|
||||
"job_code": "ops-agent-62",
|
||||
"action": "health.snapshot",
|
||||
"status": "success",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:08:00",
|
||||
"created_at": "2026-04-18 09:07:00",
|
||||
},
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"reason": "当前没有足够的执行面接管覆盖。",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh")
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"]))
|
||||
self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"])
|
||||
self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"])
|
||||
self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_includes_execution_scene_and_log_sync_items(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
detect = {
|
||||
"active_job": {
|
||||
"job_code": "detect-001",
|
||||
"latest_cycle_event": {
|
||||
"created_at": "2026-04-18 09:12:30",
|
||||
},
|
||||
},
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "代理池正常,现场可继续观察",
|
||||
"recent_event": "域名检测完成: example.com",
|
||||
"recent_warning": "",
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"mode_label": "关键回传",
|
||||
"description": "当前为关键回传;适合常态联调。",
|
||||
"line_count": 12,
|
||||
"source_node_count": 1,
|
||||
"source_nodes": ["mainland-worker-01"],
|
||||
"last_at": "2026-04-18 09:12:31",
|
||||
"preview_lines": ["worker log line"],
|
||||
},
|
||||
"participation_summary": {
|
||||
"summary": "当前参与检测节点 1 台,其中执行/已领 1 台,近窗刚有吞吐 0 台。",
|
||||
"effective_online_nodes": 1,
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": "2026-04-18 09:12:29",
|
||||
"participation_state": "dispatch_active",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [],
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(detect),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
items_by_kind = {item["kind"]: item for item in payload["items"]}
|
||||
self.assertIn("execution_scene", items_by_kind)
|
||||
self.assertIn("log_sync", items_by_kind)
|
||||
|
||||
execution_scene_item = items_by_kind["execution_scene"]
|
||||
log_sync_item = items_by_kind["log_sync"]
|
||||
|
||||
self.assertEqual("running", execution_scene_item["status"])
|
||||
self.assertEqual("执行中", execution_scene_item["status_label"])
|
||||
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"])
|
||||
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"])
|
||||
self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"])
|
||||
self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"])
|
||||
self.assertIn("执行/已领 1", execution_scene_item["meta_text"])
|
||||
self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("running", log_sync_item["status"])
|
||||
self.assertEqual("执行中", log_sync_item["status_label"])
|
||||
self.assertEqual("control-plane", log_sync_item["execution_mode"])
|
||||
self.assertEqual("控制面", log_sync_item["execution_mode_label"])
|
||||
self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"])
|
||||
self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"])
|
||||
self.assertIn("关键日志观察模式", log_sync_item["summary"])
|
||||
self.assertEqual("running", payload["summary"]["status"])
|
||||
self.assertEqual("执行中", payload["summary"]["status_label"])
|
||||
self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"])
|
||||
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
925
domain-api/tests/test_ops_service_codex_action.py
Normal file
925
domain-api/tests/test_ops_service_codex_action.py
Normal file
@@ -0,0 +1,925 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_codex_action, resolve_codex_action
|
||||
|
||||
|
||||
def _build_codex_brief_entry(
|
||||
*,
|
||||
key: str,
|
||||
executor_kind: str,
|
||||
primary_action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
target_method: str = "POST",
|
||||
sequence_key: str = "",
|
||||
primary_action_payload: dict | None = None,
|
||||
secondary_action_code: str = "",
|
||||
secondary_action_payload: dict | None = None,
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
primary_action_payload = dict(primary_action_payload or {})
|
||||
secondary_action_payload = dict(secondary_action_payload or {})
|
||||
normalized_focus_ref = dict(focus_ref or {})
|
||||
normalized_primary_focus_ref = dict(primary_focus_ref or {})
|
||||
normalized_secondary_focus_ref = dict(secondary_focus_ref or {})
|
||||
if normalized_primary_focus_ref and not dict(primary_action_payload.get("focus_ref") or {}):
|
||||
primary_action_payload["focus_ref"] = dict(normalized_primary_focus_ref)
|
||||
if normalized_secondary_focus_ref and not dict(secondary_action_payload.get("focus_ref") or {}):
|
||||
secondary_action_payload["focus_ref"] = dict(normalized_secondary_focus_ref)
|
||||
request_payload_preview = (
|
||||
{
|
||||
"sequence_key": sequence_key,
|
||||
"secondary": False,
|
||||
"action_payload": primary_action_payload,
|
||||
}
|
||||
if executor_kind == "runbook_sequence"
|
||||
else {
|
||||
"action_code": primary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": primary_action_payload,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"key": key,
|
||||
"title": "测试动作",
|
||||
"summary": "测试摘要",
|
||||
"reason": "测试原因",
|
||||
"meta_text": "meta",
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"focus_ref": normalized_focus_ref,
|
||||
"primary_focus_ref": normalized_primary_focus_ref,
|
||||
"secondary_focus_ref": normalized_secondary_focus_ref,
|
||||
"primary_label": "主动作",
|
||||
"secondary_label": "次动作" if secondary_action_code else "",
|
||||
"primary_action_code": primary_action_code,
|
||||
"secondary_action_code": secondary_action_code,
|
||||
"autopilot": {
|
||||
"status_text": "测试状态",
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"backend_handled": True,
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"reason": "autopilot reason",
|
||||
"target_method": target_method,
|
||||
"target_api": target_api,
|
||||
},
|
||||
"primary_execution": {
|
||||
"label": "主动作",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": primary_action_payload,
|
||||
"request_payload_preview": request_payload_preview,
|
||||
},
|
||||
"secondary_execution": (
|
||||
{
|
||||
"label": "次动作",
|
||||
"node_codes": [],
|
||||
"action_payload": secondary_action_payload,
|
||||
}
|
||||
if secondary_action_code
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_preview_data(
|
||||
*,
|
||||
executor_kind: str,
|
||||
action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
request_payload: dict,
|
||||
sequence_key: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"execution_chain": "ops-driver -> preview",
|
||||
"focus_ref": dict(focus_ref or {}),
|
||||
"primary_focus_ref": dict(primary_focus_ref or {}),
|
||||
"secondary_focus_ref": dict(secondary_focus_ref or {}),
|
||||
"primary": {
|
||||
"label": "主动作",
|
||||
"action_code": action_code,
|
||||
"target_api": target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(request_payload or {}),
|
||||
"focus_ref": dict(primary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
},
|
||||
"secondary": {},
|
||||
}
|
||||
|
||||
|
||||
def _build_codex_activity_focus_entry(
|
||||
*,
|
||||
key: str,
|
||||
activity_key: str,
|
||||
activity_kind: str,
|
||||
ui_intent_kind: str,
|
||||
focus_ref: dict | None = None,
|
||||
source_focus_ref: dict | None = None,
|
||||
target_node_codes: list[str] | None = None,
|
||||
) -> dict:
|
||||
normalized_focus_ref = dict(focus_ref or {})
|
||||
return {
|
||||
"key": key,
|
||||
"activity_key": activity_key,
|
||||
"activity_kind": activity_kind,
|
||||
"title": "观察焦点",
|
||||
"subtitle": "日志回传待确认",
|
||||
"summary": "该焦点属于观察项,应进入对应工作区查看上下文。",
|
||||
"status": "attention",
|
||||
"status_label": "待关注",
|
||||
"meta_text": "activity-meta",
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
"target_node_codes": list(target_node_codes or []),
|
||||
"focus_ref": normalized_focus_ref,
|
||||
"source_focus_ref": dict(source_focus_ref or {}),
|
||||
"ui_intent": {
|
||||
"kind": ui_intent_kind,
|
||||
"activity_key": activity_key,
|
||||
},
|
||||
"ui_intent_kind": ui_intent_kind,
|
||||
"focus_action_code": "focus_activity_item",
|
||||
"focus_action_payload": {
|
||||
"activity_key": activity_key,
|
||||
"kind": activity_kind,
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": ui_intent_kind,
|
||||
"activity_key": activity_key,
|
||||
},
|
||||
"focus_ref": normalized_focus_ref,
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
|
||||
|
||||
class OpsServiceCodexActionTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_returns_gate_for_safe_auto(self, mock_get_ops_codex_brief, mock_preview_driver_action) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "recommendation:enable-log-sync"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="recommendation:enable-log-sync",
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Codex 驾驶动作解析完成", message)
|
||||
self.assertEqual("recommendation:enable-log-sync", data["entry_key"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
|
||||
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
|
||||
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_requires_confirmation_for_guarded_runbook(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_action_payload={"release_id": 7},
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
self.assertFalse(data["gate"]["will_execute"])
|
||||
self.assertTrue(data["gate"]["confirm_required"])
|
||||
self.assertEqual("release_progression", data["gate"]["sequence_key"])
|
||||
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_can_select_secondary_section(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_action_payload={"release_id": 7},
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
preview_data = _build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
preview_data["secondary"] = {
|
||||
"label": "查看版本区",
|
||||
"action_code": "focus_release_hub",
|
||||
"target_api": "/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
"target_method": "POST",
|
||||
"request_payload": {
|
||||
"secondary": True,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"execution_profile": {
|
||||
"action_code": "focus_release_hub",
|
||||
"backend_handled": False,
|
||||
"automation_level": "safe_auto",
|
||||
"recommendation": "auto_execute",
|
||||
"executor_mode_hint": "ui-intent",
|
||||
"risk_level": "low",
|
||||
"confirm_required": False,
|
||||
"ui_only": False,
|
||||
"blocked": False,
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "secondary preview reason",
|
||||
},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (True, "ok", preview_data)
|
||||
|
||||
ok, _, data = resolve_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
"secondary": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("secondary", data["selected_section"])
|
||||
self.assertTrue(data["gate"]["secondary"])
|
||||
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
|
||||
self.assertEqual("auto_execute", data["gate"]["recommendation"])
|
||||
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", data["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_can_fall_back_to_activity_focus_entry(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
_build_codex_activity_focus_entry(
|
||||
key="activity:ops-job-42",
|
||||
activity_key="ops-job-42",
|
||||
activity_kind="ops_job",
|
||||
ui_intent_kind="job_events",
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
source_focus_ref={
|
||||
"kind": "ops_job_event",
|
||||
"job_key": "job-42",
|
||||
"event_key": "event-1",
|
||||
},
|
||||
target_node_codes=["mainland-controller-01"],
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_activity_item",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_activity_item",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"activity_key": "ops-job-42",
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"activity_key": "ops-job-42",
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("activity:ops-job-42", data["entry_key"])
|
||||
self.assertTrue(data["selected_entry"]["observation_only"])
|
||||
self.assertEqual("ops_job", data["selected_entry"]["activity_kind"])
|
||||
self.assertEqual("focus_activity_item", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("job-42", data["focus_ref"]["job_key"])
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_safe_auto_driver_action(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "recommendation:enable-log-sync"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="recommendation:enable-log-sync",
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
)
|
||||
],
|
||||
}
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload=request_payload,
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"执行成功",
|
||||
{"handled": True, "action_code": "enable_log_sync_key"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_driver_action.assert_called_once_with(request_payload)
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_blocks_guarded_runbook_without_confirm(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("确认", message)
|
||||
self.assertFalse(data["executed"])
|
||||
mock_execute_ops_runbook_sequence.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_activity_focus_as_safe_auto(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
_build_codex_activity_focus_entry(
|
||||
key="activity:ops-job-42",
|
||||
activity_key="ops-job-42",
|
||||
activity_kind="ops_job",
|
||||
ui_intent_kind="job_events",
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_activity_item",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_activity_item",
|
||||
"node_codes": [],
|
||||
"action_payload": {
|
||||
"activity_key": "ops-job-42",
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"activity_key": "ops-job-42",
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"执行成功",
|
||||
{"handled": True, "action_code": "focus_activity_item"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_driver_action.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_supports_focus_latest_job_events_without_ui_intent(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-81", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
{
|
||||
"key": "activity:ops-job-81",
|
||||
"activity_key": "ops-job-81",
|
||||
"activity_kind": "ops_job",
|
||||
"title": "开始回执异常",
|
||||
"subtitle": "ops-agent-81",
|
||||
"summary": "节点已接单执行,但开始回执未成功送达控制面。",
|
||||
"status": "attention",
|
||||
"status_label": "待处理",
|
||||
"meta_text": "ops-agent-81",
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"ui_intent": {},
|
||||
"ui_intent_kind": "",
|
||||
"focus_action_code": "focus_latest_job_events",
|
||||
"focus_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_latest_job_events",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_latest_job_events",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("activity:ops-job-81", data["entry_key"])
|
||||
self.assertEqual("focus_latest_job_events", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_guarded_runbook_after_confirm(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
)
|
||||
],
|
||||
}
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
mock_execute_ops_runbook_sequence.return_value = (
|
||||
True,
|
||||
"runbook 执行成功",
|
||||
{"sequence_key": "release_progression", "driver_action_code": "create_release_rollout_worker"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
"confirm": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("runbook 执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
497
domain-api/tests/test_ops_service_codex_brief.py
Normal file
497
domain-api/tests/test_ops_service_codex_brief.py
Normal file
@@ -0,0 +1,497 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_codex_brief
|
||||
|
||||
|
||||
class OpsServiceCodexBriefTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_classifies_safe_ui_and_runbook_entries(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前建议已形成。",
|
||||
"go_live_summary": {
|
||||
"go_live_status": "attention",
|
||||
"publish_ready": False,
|
||||
"publish_status": "attention",
|
||||
"publish_status_label": "可发布但建议先复核",
|
||||
"operator_title": "先补现场日志回传",
|
||||
"next_step_action_code": "enable_log_sync_key",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 1,
|
||||
},
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "recommendation:enable-scene-log-sync",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先补现场日志回传",
|
||||
"summary": "优先打开关键日志回传。",
|
||||
"reason": "当前还看不到现场过程日志。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"meta_text": "",
|
||||
"detail_lines": ["参与节点 2 台"],
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs.full",
|
||||
},
|
||||
"primary_label": "开启关键回传",
|
||||
"secondary_label": "开启全量回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"secondary_action_code": "enable_log_sync_full",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"secondary_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs.full",
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "recommendation:release-missing",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先建立 Release",
|
||||
"summary": "当前还没有默认 Release。",
|
||||
"reason": "需要先进入版本区。",
|
||||
"status": "planned",
|
||||
"level_label": "待补齐",
|
||||
"tag_type": "info",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": [],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "创建 Release",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "open_release_dialog",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
{
|
||||
"key": "runbook:release_progression",
|
||||
"kind": "runbook_sequence",
|
||||
"lane": "standard_path",
|
||||
"lane_label": "标准路径",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"status": "attention",
|
||||
"level_label": "待处理",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "Release / Rollout",
|
||||
"detail_lines": ["当前解析:Worker 灰度"],
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"primary_label": "Worker 灰度",
|
||||
"secondary_label": "查看版本区",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_node_codes": ["mainland-controller-01"],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
"secondary_action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual(3, payload["summary"]["total"])
|
||||
self.assertEqual("attention", payload["go_live_summary"]["go_live_status"])
|
||||
self.assertEqual("attention", payload["summary"]["go_live_status"])
|
||||
self.assertEqual("先补现场日志回传", payload["summary"]["operator_title"])
|
||||
self.assertEqual("attention", payload["summary"]["publish_status"])
|
||||
self.assertFalse(payload["summary"]["publish_ready"])
|
||||
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["summary"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("当前建议已形成。", payload["driver_feed"]["headline"])
|
||||
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual(2, payload["summary"]["auto_execute_total"])
|
||||
self.assertEqual(1, payload["summary"]["confirm_then_execute_total"])
|
||||
self.assertEqual(0, payload["summary"]["open_ui_total"])
|
||||
self.assertEqual(1, payload["summary"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["summary"]["backend_handled_total"])
|
||||
self.assertEqual("attention", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("recommendation:enable-scene-log-sync", payload["focus"]["entry_key"])
|
||||
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", payload["focus"]["primary_focus_ref"]["scene_key"])
|
||||
self.assertEqual("auto_execute", payload["entries"][0]["autopilot"]["recommendation"])
|
||||
self.assertIn("ops_observability_contract", payload["entries"][0]["contract_keys"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["entries"][0]["contract_keys"])
|
||||
self.assertEqual("scene.logs", payload["entries"][0]["primary_focus_ref"]["scene_key"])
|
||||
self.assertEqual("safe_auto", payload["entries"][1]["autopilot"]["automation_level"])
|
||||
self.assertIn("release_hub_contract", payload["entries"][1]["contract_keys"])
|
||||
|
||||
runbook_entry = payload["entries"][2]
|
||||
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
|
||||
self.assertEqual("confirm_then_execute", runbook_entry["autopilot"]["recommendation"])
|
||||
self.assertIn("ops_playbook_contract", runbook_entry["contract_keys"])
|
||||
self.assertIn("release_hub_contract", runbook_entry["contract_keys"])
|
||||
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
|
||||
self.assertEqual("worker", runbook_entry["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
|
||||
self.assertEqual(
|
||||
"/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
runbook_entry["autopilot"]["target_api"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"sequence_key": "release_progression",
|
||||
"secondary": False,
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
runbook_entry["primary_execution"]["request_payload_preview"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"sequence_key": "release_progression",
|
||||
"secondary": True,
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
runbook_entry["secondary_execution"]["request_payload_preview"],
|
||||
)
|
||||
self.assertEqual("attention", payload["automation_coverage"]["launch_status"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["guarded_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_marks_mixed_and_blocked_entries(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前建议需要先复核。",
|
||||
"go_live_summary": {
|
||||
"go_live_status": "blocked",
|
||||
"publish_ready": False,
|
||||
"publish_status": "blocked",
|
||||
"publish_status_label": "暂不可发布",
|
||||
"operator_title": "先补执行面接管",
|
||||
"next_step_action_code": "fix_managed_nodes",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "补执行面接管",
|
||||
"launchpad_recommended_recovery_summary": "先处理 launchpad 缺口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "recommendation:release-launchpad-fix-managed",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先补执行面接管",
|
||||
"summary": "先处理 launchpad 缺口。",
|
||||
"reason": "当前动作可能直接 sync,也可能转为补接入。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 9,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 9,
|
||||
"section": "default_rollout_gate",
|
||||
},
|
||||
"primary_label": "补执行面接管",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": ["mainland-controller-01"],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {"channel": "stable"},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
{
|
||||
"key": "recommendation:inspect-participating",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先看执行现场",
|
||||
"summary": "先跑参与节点巡检。",
|
||||
"reason": "缺少目标节点时不应误判为可自动执行。",
|
||||
"status": "attention",
|
||||
"level_label": "推荐",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": [],
|
||||
"executor_kind": "driver_action",
|
||||
"primary_label": "执行标准巡检",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("blocked", payload["summary"]["publish_status"])
|
||||
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("补执行面接管", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["resolve_first_total"])
|
||||
self.assertEqual(1, payload["summary"]["blocked_total"])
|
||||
self.assertEqual("blocked", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("mixed", payload["entries"][0]["autopilot"]["automation_level"])
|
||||
self.assertEqual("resolve_first", payload["entries"][0]["autopilot"]["recommendation"])
|
||||
self.assertEqual("default_rollout_gate", payload["entries"][0]["primary_focus_ref"]["section"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["blocked_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["human_dependency_total"])
|
||||
|
||||
blocked_entry = payload["entries"][1]
|
||||
self.assertTrue(blocked_entry["autopilot"]["blocked"])
|
||||
self.assertEqual(["node_codes"], blocked_entry["primary_execution"]["missing_fields"])
|
||||
self.assertEqual("blocked", blocked_entry["autopilot"]["recommendation"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_exposes_activity_focus_and_uses_it_as_fallback_focus(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前先盯住执行现场。",
|
||||
"scene_log_observation": {},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
{
|
||||
"key": "activity:execution-scene:live",
|
||||
"activity_key": "execution-scene:live",
|
||||
"activity_kind": "execution_scene",
|
||||
"title": "当前执行现场",
|
||||
"subtitle": "运行中",
|
||||
"summary": "当前参与检测节点 2 台,建议先看现场与日志回传。",
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"meta_text": "执行/已领 2 / 待命 1",
|
||||
"occurred_at": "2026-04-18 09:30:00",
|
||||
"target_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"source_focus_ref": {},
|
||||
"ui_intent": {
|
||||
"kind": "focus_execution_scene",
|
||||
"open_log_drawer": True,
|
||||
},
|
||||
"ui_intent_kind": "focus_execution_scene",
|
||||
"focus_action_code": "focus_activity_item",
|
||||
"focus_action_payload": {
|
||||
"activity_key": "execution-scene:live",
|
||||
"kind": "execution_scene",
|
||||
"status": "running",
|
||||
"ui_intent": {
|
||||
"kind": "focus_execution_scene",
|
||||
"open_log_drawer": True,
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual(0, payload["summary"]["total"])
|
||||
self.assertEqual(1, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("activity:execution-scene:live", payload["focus"]["entry_key"])
|
||||
self.assertEqual("open_ui", payload["focus"]["recommendation"])
|
||||
self.assertIn("ops_observability_contract", payload["focus"]["contract_keys"])
|
||||
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual(1, len(payload["activity_focus"]))
|
||||
self.assertEqual("focus_execution_scene", payload["activity_focus"][0]["ui_intent_kind"])
|
||||
self.assertEqual(
|
||||
"ops_driver_contract",
|
||||
payload["activity_focus"][0]["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
self.assertTrue(payload["activity_focus"][0]["observation_only"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_can_fallback_to_scene_log_observation_focus(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "先看现场日志。",
|
||||
"entries": [],
|
||||
"activity_focus": [],
|
||||
"scene_log_observation": {
|
||||
"status": "waiting_sample",
|
||||
"status_label": "等待样本",
|
||||
"summary": "日志回传已开启,但样本还没形成。",
|
||||
"focus_ref": {
|
||||
"kind": "node_scene_log",
|
||||
"node_code": "mainland-worker-01",
|
||||
"mode": "key",
|
||||
"limit": 120,
|
||||
},
|
||||
"contract_keys": ["ops_observability_contract", "ops_stack_diagnosis_contract"],
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("scene-log-observation", payload["focus"]["entry_key"])
|
||||
self.assertEqual("open_ui", payload["focus"]["recommendation"])
|
||||
self.assertEqual("node_scene_log", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual("mainland-worker-01", payload["focus"]["focus_ref"]["node_code"])
|
||||
self.assertEqual("waiting_sample", payload["scene_log_observation"]["status"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_marks_bootstrap_run_as_agent_protocol_flow(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "先执行节点接入收口。",
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "priority:handover-gap-bootstrap_run",
|
||||
"kind": "priority_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "当前优先",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "Launchpad 已明确给出首台节点的接入收口动作。",
|
||||
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "overview.recommendation",
|
||||
"detail_lines": ["当前已收敛到单节点接入动作。"],
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01", "section": "bootstrap_plan"},
|
||||
"primary_label": "跑接入收口",
|
||||
"secondary_label": "查看 Bootstrap 计划",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"secondary_action_code": "create_bootstrap_plan",
|
||||
"primary_node_codes": ["mainland-worker-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
"secondary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("priority:handover-gap-bootstrap_run", payload["focus"]["entry_key"])
|
||||
self.assertEqual("resolve_first", payload["focus"]["recommendation"])
|
||||
self.assertIn("ops_agent_protocol", payload["entries"][0]["contract_keys"])
|
||||
self.assertEqual("bootstrap_run", payload["entries"][0]["primary_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["entries"][0]["primary_execution"]["node_codes"][0])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", payload["entries"][0]["autopilot"]["target_api"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
295
domain-api/tests/test_ops_service_contract_registry.py
Normal file
295
domain-api/tests/test_ops_service_contract_registry.py
Normal file
@@ -0,0 +1,295 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import (
|
||||
get_ops_blueprint,
|
||||
get_ops_capabilities,
|
||||
get_ops_contract_detail,
|
||||
get_ops_contract_registry,
|
||||
get_ops_runbook,
|
||||
)
|
||||
|
||||
|
||||
class OpsServiceContractRegistryTests(unittest.TestCase):
|
||||
def test_contract_registry_lists_all_frozen_ops_contracts(self) -> None:
|
||||
payload = get_ops_contract_registry()
|
||||
|
||||
self.assertEqual("2026-04-18", payload["registry_version"])
|
||||
self.assertEqual("v1", payload["schema_version"])
|
||||
self.assertEqual(11, payload["contracts_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"ops_job_contract",
|
||||
"ops_agent_protocol",
|
||||
"release_hub_contract",
|
||||
"ops_driver_contract",
|
||||
"ops_playbook_contract",
|
||||
"ops_observability_contract",
|
||||
"ops_stack_diagnosis_contract",
|
||||
"ops_doctor_decision_contract",
|
||||
"ops_go_live_signoff_contract",
|
||||
"ops_go_live_bundle_contract",
|
||||
"ops_go_live_review_contract",
|
||||
],
|
||||
[item["key"] for item in payload["contracts"]],
|
||||
)
|
||||
self.assertIn("/api/v1/ops/contracts/{contract_key}", payload["discovery_entrypoints"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_job_contract.md",
|
||||
payload["contracts_by_key"]["ops_job_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/jobs/{job_id}/dispatch",
|
||||
payload["contracts_by_key"]["ops_job_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_agent_protocol.md",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/agent/bootstrap-plan",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"/api/v1/ops/releases/launchpad",
|
||||
payload["contracts_by_key"]["release_hub_contract"]["primary_endpoint"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-brief",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/preview",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/resolve",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/execute-resolved",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-actions/resolve",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-actions/execute",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_playbook_contract.md",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/playbooks/execute",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/playbook-runs/{run_code}/events",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_observability_contract.md",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/inspection-overview",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/scene-log",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_stack_diagnosis_contract.md",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/stack-diagnosis",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_driver_contract",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_doctor_decision_contract.md",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/doctor-decision",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_signoff_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-signoff",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_bundle_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-bundle",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_go_live_signoff_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_review_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-review",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_go_live_bundle_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_doctor_decision_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
|
||||
)
|
||||
|
||||
def test_contract_detail_exposes_related_contracts(self) -> None:
|
||||
payload = get_ops_contract_detail("ops_stack_diagnosis_contract")
|
||||
|
||||
self.assertEqual("2026-04-18", payload["registry_version"])
|
||||
self.assertEqual("/api/v1/ops/contracts/ops_stack_diagnosis_contract", payload["detail_endpoint"])
|
||||
self.assertEqual("ops_stack_diagnosis_contract", payload["contract"]["key"])
|
||||
self.assertIn("ops_driver_contract", payload["related_contract_keys"])
|
||||
self.assertIn("release_hub_contract", payload["related_contract_keys"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_driver_contract.md",
|
||||
next(
|
||||
item["schema_doc_path"]
|
||||
for item in payload["related_contracts"]
|
||||
if item["key"] == "ops_driver_contract"
|
||||
),
|
||||
)
|
||||
self.assertEqual({}, get_ops_contract_detail("missing-contract"))
|
||||
|
||||
def test_capabilities_blueprint_and_runbook_expose_contract_registry(self) -> None:
|
||||
capabilities = get_ops_capabilities()
|
||||
contract_registry = capabilities["contract_registry"]
|
||||
|
||||
self.assertEqual("/api/v1/ops/contracts", contract_registry["endpoint"])
|
||||
self.assertEqual("/api/v1/ops/contracts/{contract_key}", contract_registry["detail_endpoint_pattern"])
|
||||
self.assertEqual(11, contract_registry["contracts_total"])
|
||||
self.assertIn("ops_job_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_driver_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_playbook_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_observability_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_doctor_decision_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_signoff_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_bundle_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_review_contract", contract_registry["contract_keys"])
|
||||
remote_log_capability = next(
|
||||
item for item in capabilities["capabilities"] if item["key"] == "remote_log_sync"
|
||||
)
|
||||
self.assertEqual("ready", remote_log_capability["current_state"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/release_hub_contract.md",
|
||||
contract_registry["schema_docs"]["release_hub_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_playbook_contract.md",
|
||||
contract_registry["schema_docs"]["ops_playbook_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_job_contract.md",
|
||||
contract_registry["schema_docs"]["ops_job_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_observability_contract.md",
|
||||
contract_registry["schema_docs"]["ops_observability_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_stack_diagnosis_contract.md",
|
||||
contract_registry["schema_docs"]["ops_stack_diagnosis_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_doctor_decision_contract.md",
|
||||
contract_registry["schema_docs"]["ops_doctor_decision_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_signoff_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_signoff_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_bundle_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_bundle_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_review_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_review_contract"],
|
||||
)
|
||||
|
||||
blueprint = get_ops_blueprint()
|
||||
self.assertEqual("2026-04-18", blueprint["contract_registry"]["registry_version"])
|
||||
self.assertEqual(11, blueprint["contract_registry"]["contracts_total"])
|
||||
|
||||
with patch("app.services.ops_service.get_runtime_status") as mock_runtime_status, patch(
|
||||
"app.services.ops_service.list_managed_nodes_with_agent_state"
|
||||
) as mock_list_managed_nodes_with_agent_state, patch(
|
||||
"app.services.ops_service.get_release_launchpad"
|
||||
) as mock_get_release_launchpad, patch(
|
||||
"app.services.ops_service._attach_ops_runbook_sequence_resolutions"
|
||||
) as mock_attach_sequences:
|
||||
mock_runtime_status.return_value = {
|
||||
"readiness": {"status": "ready", "summary": "ok"},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {}
|
||||
mock_attach_sequences.return_value = []
|
||||
runbook = get_ops_runbook()
|
||||
|
||||
self.assertIn("GET /api/v1/ops/contracts", runbook["fast_checks"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
149
domain-api/tests/test_ops_service_doctor_decision.py
Normal file
149
domain-api/tests/test_ops_service_doctor_decision.py
Normal file
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_doctor_decision
|
||||
|
||||
|
||||
class OpsServiceDoctorDecisionTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service._build_live_doctor_payload")
|
||||
@patch("app.services.ops_service._find_latest_go_live_bundle_manifest")
|
||||
def test_doctor_decision_falls_back_to_live_payload_when_bundle_missing(
|
||||
self,
|
||||
mock_find_latest_go_live_bundle_manifest,
|
||||
mock_build_live_doctor_payload,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_find_latest_go_live_bundle_manifest.return_value = (None, Path("/tmp/go-live-bundles"))
|
||||
mock_build_live_doctor_payload.return_value = (
|
||||
{
|
||||
"status": "attention",
|
||||
"reason_code": "live_fallback",
|
||||
"headline": "当前先补 attention 项",
|
||||
"detail": "需要先顺着 stack next 收口。",
|
||||
"preferred_surface": "stack-summary",
|
||||
"next_action_code": "review_stack_next",
|
||||
"recommended_commands": ["bash primary"],
|
||||
"evidence": {},
|
||||
},
|
||||
{
|
||||
"ok": False,
|
||||
"required_failures": [],
|
||||
"optional_unavailable": [],
|
||||
"scene_log_reports_total": 0,
|
||||
"scene_log_reports_ok": 0,
|
||||
"scene_log_status_counts": {},
|
||||
"scene_log_reports": [],
|
||||
"contract_surface_gaps": [],
|
||||
},
|
||||
)
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_doctor_decision_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertEqual("主决策待复核", payload["status_label"])
|
||||
self.assertEqual("live_fallback", payload["source"])
|
||||
self.assertEqual("review_stack_next", payload["decision"]["next_action_code"])
|
||||
self.assertIn("doctor-decision", payload["recommended_commands"]["doctor_decision"])
|
||||
self.assertEqual("bash primary", payload["recommended_commands"]["primary_recovery"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
|
||||
payload["recommended_commands"]["stack_check"],
|
||||
)
|
||||
self.assertEqual(
|
||||
payload["recommended_commands"]["stack_check"],
|
||||
payload["recommended_commands"]["stack_diagnosis"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_doctor_decision_reads_bundle_doctor_artifact_when_available(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_doctor_decision_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
doctor_artifact_path = report_dir / "07_doctor_decision.json"
|
||||
doctor_artifact_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"summary": {
|
||||
"ok": False,
|
||||
"required_failures": ["06_contracts_registry.txt"],
|
||||
"optional_unavailable": ["12_activity_preview.txt"],
|
||||
"scene_log_reports_total": 2,
|
||||
"scene_log_reports_ok": 1,
|
||||
"scene_log_status_counts": {"ok": 1, "missing": 1},
|
||||
"scene_log_reports": [
|
||||
{"node_code": "mainland-worker-01", "status": "ok", "ok": True},
|
||||
{"node_code": "mainland-controller-01", "status": "missing", "ok": False},
|
||||
],
|
||||
"contract_surface_gaps": ["activity-stream"],
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补协议面",
|
||||
"launchpad_recommended_recovery_summary": "先补 driver / activity 的协议导航。",
|
||||
},
|
||||
"decision": {
|
||||
"status": "blocked",
|
||||
"reason_code": "contracts_registry_unavailable",
|
||||
"headline": "contract registry 当前不可用",
|
||||
"detail": "先恢复 contracts registry 再继续自动驾驶。",
|
||||
"preferred_surface": "contracts",
|
||||
"next_action_code": "restore_contract_registry",
|
||||
"recommended_commands": [
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100",
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
],
|
||||
"evidence": {"surface_http_status": {"contracts_registry": 503}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "doctor_decision", "path": str(doctor_artifact_path), "ok": True},
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("blocked", payload["status"])
|
||||
self.assertEqual("bundle_manifest", payload["source"])
|
||||
self.assertEqual("contracts", payload["decision"]["preferred_surface"])
|
||||
self.assertEqual(["06_contracts_registry.txt"], payload["summary"]["required_failures"])
|
||||
self.assertEqual(["mainland-controller-01"], payload["summary"]["scene_log_problem_node_codes"])
|
||||
self.assertIn("contracts", payload["recommended_commands"]["primary_recovery"])
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["doctor_decision"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
274
domain-api/tests/test_ops_service_driver_action_preview.py
Normal file
274
domain-api/tests/test_ops_service_driver_action_preview.py
Normal file
@@ -0,0 +1,274 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import preview_driver_action
|
||||
|
||||
|
||||
class OpsServiceDriverActionPreviewTests(unittest.TestCase):
|
||||
def test_preview_driver_action_builds_safe_auto_contract_preview(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "先补现场日志回传",
|
||||
"summary": "优先打开关键日志回传。",
|
||||
"reason": "当前还看不到现场过程日志。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_label": "开启关键回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
self.assertEqual("ops_driver_contract", data["contract_key"])
|
||||
self.assertEqual("/api/v1/ops/driver-feed", data["contract_primary_endpoint"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/preview", data["preview_endpoint"])
|
||||
self.assertEqual("driver_action", data["executor_kind"])
|
||||
self.assertEqual(
|
||||
"ops-driver -> driver-action execute -> playbook / ui-intent / settings",
|
||||
data["execution_chain"],
|
||||
)
|
||||
|
||||
primary = data["primary"]
|
||||
self.assertEqual("开启关键回传", primary["label"])
|
||||
self.assertEqual("enable_log_sync_key", primary["action_code"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertEqual("execution_scene", data["focus_ref"]["kind"])
|
||||
self.assertEqual("execution_scene", data["primary_focus_ref"]["kind"])
|
||||
self.assertEqual("execution_scene", primary["focus_ref"]["kind"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": [],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
},
|
||||
primary["request_payload"],
|
||||
)
|
||||
self.assertFalse(data["secondary"])
|
||||
|
||||
def test_preview_driver_action_marks_missing_required_fields_as_blocked(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "打开全量现场取证",
|
||||
"summary": "当前需要进入取证工作区。",
|
||||
"reason": "需要明确 playbook 上下文。",
|
||||
"executor_kind": "driver_action",
|
||||
"primary_label": "打开取证",
|
||||
"primary_action_code": "open_playbook_dialog",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("当前阻断", primary["recommendation_label"])
|
||||
self.assertTrue(primary["execution_profile"]["blocked"])
|
||||
self.assertEqual(["playbook_key"], primary["execution_profile"]["missing_fields"])
|
||||
|
||||
def test_preview_driver_action_marks_release_launchpad_review_as_safe_auto(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "查看 Worker 预案",
|
||||
"summary": "当前建议先复核 Worker launchpad 预案。",
|
||||
"reason": "需要先判断目标节点、门禁和阻断缺口。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 88,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 88,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_label": "查看 Worker 预案",
|
||||
"primary_action_code": "review_smart_rollout_preview",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("review_smart_rollout_preview", primary["action_code"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertFalse(primary["execution_profile"]["ui_only"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
|
||||
|
||||
def test_preview_driver_action_marks_handover_gap_as_safe_auto(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "当前存在待接管节点。",
|
||||
"reason": "需要先查看首台缺口节点的接管详情。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "managed_node",
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "managed_node",
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"primary_label": "查看接管详情",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"primary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("handover_first_gap", primary["action_code"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertFalse(primary["execution_profile"]["ui_only"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_preview_driver_action_supports_runbook_sequence_primary_and_secondary(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"target_scope_label": "Release / Rollout / 2026.04.18",
|
||||
"step_titles": ["确认发布包", "发灰度"],
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "Worker 灰度",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_label": "查看版本区",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"secondary_action_payload": {"release_id": 7},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_label": "Worker 灰度",
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
self.assertEqual("runbook_sequence", data["executor_kind"])
|
||||
self.assertEqual("runbook sequence", data["executor_kind_label"])
|
||||
self.assertEqual(
|
||||
"ops-driver -> runbook-sequence -> resolved driver action / ui-intent",
|
||||
data["execution_chain"],
|
||||
)
|
||||
self.assertEqual("create_release_rollout_worker", data["resolved_primary"]["driver_action_code"])
|
||||
self.assertEqual("release_launchpad", data["focus_ref"]["section"])
|
||||
self.assertEqual("worker", data["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", data["secondary_focus_ref"]["section"])
|
||||
|
||||
primary = data["primary"]
|
||||
self.assertEqual("/api/v1/ops/runbook/sequences/release_progression/execute", primary["target_api"])
|
||||
self.assertEqual("建议确认后执行", primary["recommendation_label"])
|
||||
self.assertEqual("guarded_auto", primary["automation_level_label"])
|
||||
self.assertEqual("worker", primary["focus_ref"]["mode"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"secondary": False,
|
||||
"requested_by": "web-ui/ops-center",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
primary["request_payload"],
|
||||
)
|
||||
|
||||
secondary = data["secondary"]
|
||||
self.assertEqual("查看版本区", secondary["label"])
|
||||
self.assertEqual("focus_release_hub", secondary["action_code"])
|
||||
self.assertEqual("release_detail", secondary["focus_ref"]["section"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "web-ui/ops-center",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
secondary["request_payload"],
|
||||
)
|
||||
self.assertEqual("可自动执行", secondary["recommendation_label"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
433
domain-api/tests/test_ops_service_driver_action_resolve.py
Normal file
433
domain-api/tests/test_ops_service_driver_action_resolve.py
Normal file
@@ -0,0 +1,433 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_resolved_driver_action, resolve_driver_action_request
|
||||
|
||||
|
||||
def _build_preview_data(
|
||||
*,
|
||||
executor_kind: str,
|
||||
primary_action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
primary_request_payload: dict,
|
||||
sequence_key: str = "",
|
||||
secondary_action_code: str = "",
|
||||
secondary_request_payload: dict | None = None,
|
||||
secondary_recommendation: str = "",
|
||||
secondary_automation_level: str = "",
|
||||
secondary_target_api: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"execution_chain": "ops-driver -> preview",
|
||||
"focus_ref": dict(focus_ref or {}),
|
||||
"primary_focus_ref": dict(primary_focus_ref or {}),
|
||||
"secondary_focus_ref": dict(secondary_focus_ref or {}),
|
||||
"primary": {
|
||||
"label": "主动作",
|
||||
"action_code": primary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"target_api": target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(primary_request_payload or {}),
|
||||
"focus_ref": dict(primary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": primary_action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
},
|
||||
"secondary": (
|
||||
{
|
||||
"label": "次动作",
|
||||
"action_code": secondary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"target_api": secondary_target_api or target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(secondary_request_payload or {}),
|
||||
"focus_ref": dict(secondary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": secondary_action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": secondary_automation_level or automation_level,
|
||||
"recommendation": secondary_recommendation or recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if (secondary_recommendation or recommendation) == "auto_execute" else "high",
|
||||
"confirm_required": (secondary_recommendation or recommendation) == "confirm_then_execute",
|
||||
"ui_only": (secondary_recommendation or recommendation) == "open_ui",
|
||||
"blocked": (secondary_recommendation or recommendation) == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
}
|
||||
if secondary_action_code
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class OpsServiceDriverActionResolveTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_returns_gate_for_safe_auto(self, mock_preview_driver_action) -> None:
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
primary_request_payload=request_payload,
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "开启关键日志回传",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("驾驶动作解析完成", message)
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
|
||||
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
|
||||
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_requires_confirmation_for_guarded_runbook(self, mock_preview_driver_action) -> None:
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
self.assertFalse(data["gate"]["will_execute"])
|
||||
self.assertTrue(data["gate"]["confirm_required"])
|
||||
self.assertEqual("release_progression", data["gate"]["sequence_key"])
|
||||
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_can_select_secondary_open_ui_section(self, mock_preview_driver_action) -> None:
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_request_payload={
|
||||
"secondary": True,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
secondary_recommendation="auto_execute",
|
||||
secondary_automation_level="safe_auto",
|
||||
secondary_target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"secondary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
"secondary": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("secondary", data["selected_section"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
|
||||
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", data["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_blocks_guarded_action_without_confirm(self, mock_preview_driver_action) -> None:
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("需要显式确认", message)
|
||||
self.assertFalse(data["executed"])
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_runs_safe_auto_driver_action(
|
||||
self,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
primary_request_payload=request_payload,
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"日志回传已开启",
|
||||
{"handled": True, "mode": "ops-job"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "开启关键日志回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("日志回传已开启", message)
|
||||
self.assertTrue(data["executed"])
|
||||
self.assertEqual({"handled": True, "mode": "ops-job"}, data["execution_result"])
|
||||
mock_execute_driver_action.assert_called_once_with(request_payload)
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_runs_guarded_runbook_after_confirm(
|
||||
self,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
mock_execute_ops_runbook_sequence.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "smart-release-rollout",
|
||||
"driver_action_code": "create_release_rollout_worker",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
"confirm": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["executed"])
|
||||
self.assertEqual("execute_confirmed", data["gate"]["decision"])
|
||||
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1265
domain-api/tests/test_ops_service_driver_actions.py
Normal file
1265
domain-api/tests/test_ops_service_driver_actions.py
Normal file
File diff suppressed because it is too large
Load Diff
358
domain-api/tests/test_ops_service_driver_feed.py
Normal file
358
domain-api/tests/test_ops_service_driver_feed.py
Normal file
@@ -0,0 +1,358 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_driver_feed
|
||||
|
||||
|
||||
class OpsServiceDriverFeedTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_driver_feed_merges_priority_recommendation_driver_cards_and_runbook_sequences(
|
||||
self,
|
||||
mock_get_ops_overview,
|
||||
mock_get_ops_runbook,
|
||||
mock_get_ops_go_live_summary,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "blocked",
|
||||
"publish_ready": False,
|
||||
"publish_status": "blocked",
|
||||
"publish_status_label": "暂不可发布",
|
||||
"publish_summary": "当前仍有硬阻断项,不能直接进入正式发版。",
|
||||
"operator_title": "先补接管缺口",
|
||||
"next_step_action_code": "handover_first_gap",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 1,
|
||||
"blocking_reasons": ["remote_agent_ready=0"],
|
||||
"warnings": ["log_sync_waiting_sample=0/2"],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"participating_nodes": 2,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
},
|
||||
"participating_nodes": [
|
||||
{"node_code": "mainland-worker-01"},
|
||||
{"node_code": "mainland-controller-01"},
|
||||
],
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
},
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "fix-handover-gap",
|
||||
"priority": "先补接管缺口",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "当前应该先把接管链路补齐。",
|
||||
"reason": "否则后续批量动作无法稳定下发。",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"primary_label": "纳管首台节点",
|
||||
"secondary_label": "查看目标节点",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"secondary_action_code": "view_first_gap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"primary_node_codes": ["mainland-worker-02"],
|
||||
"secondary_node_codes": ["mainland-worker-02"],
|
||||
"primary_action_payload": {"requested_by": "test"},
|
||||
"secondary_action_payload": {},
|
||||
"reasons": ["当前存在接管缺口。"],
|
||||
},
|
||||
"activity_stream": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"activity_key": "ops-job:501",
|
||||
"title": "logs.collect",
|
||||
"subtitle": "ops-logs-501",
|
||||
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"summary": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"meta_text": "ops-logs-501 / 执行 远端 Agent",
|
||||
"occurred_at": "2026-04-18 09:18:00",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 501,
|
||||
"job_code": "ops-logs-501",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 501,
|
||||
"event_key": "job-event:501",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 501,
|
||||
"job_code": "ops-logs-501",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"driver_recommendations": [
|
||||
{
|
||||
"key": "fix-handover-gap",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "重复建议,不应再次进入 feed。",
|
||||
"reason": "duplicate",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"primary_label": "纳管首台节点",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
{
|
||||
"key": "inspect-scene-log-gap",
|
||||
"title": "排查现场日志样本缺口",
|
||||
"summary": "现场日志样本还没有形成。",
|
||||
"reason": "先看 Worker 日志。",
|
||||
"meta_text": "参与节点 2 台 / 已回传来源 0 台",
|
||||
"level_label": "推荐",
|
||||
"tag_type": "warning",
|
||||
"primary_label": "看 Worker 日志",
|
||||
"secondary_label": "执行标准巡检",
|
||||
"primary_action_code": "open_worker_logs",
|
||||
"secondary_action_code": "run_standard_inspection",
|
||||
"node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_action_payload": {"lines": 120},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可进入发布路径。",
|
||||
"reason": "正式运维时应统一落回 Release / Rollout。",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"tag_type": "warning",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "补执行面接管",
|
||||
"secondary_label": "查看版本区",
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "default_rollout_gate",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"step_titles": ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入"],
|
||||
"primary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "fix_managed_nodes",
|
||||
"driver_action_label": "补执行面接管",
|
||||
"driver_node_codes": ["mainland-controller-01"],
|
||||
"driver_action_payload": {"channel": "stable"},
|
||||
"resolved_at": "2026-04-18T03:00:00",
|
||||
},
|
||||
"secondary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "focus_release_hub",
|
||||
"driver_action_label": "查看版本区",
|
||||
"driver_node_codes": [],
|
||||
"driver_action_payload": {"release_id": 7},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_driver_feed()
|
||||
|
||||
self.assertEqual(3, payload["summary"]["total"])
|
||||
self.assertEqual("blocked", payload["go_live_summary"]["go_live_status"])
|
||||
self.assertEqual("blocked", payload["summary"]["go_live_status"])
|
||||
self.assertEqual("先补接管缺口", payload["summary"]["operator_title"])
|
||||
self.assertEqual("blocked", payload["summary"]["publish_status"])
|
||||
self.assertFalse(payload["summary"]["publish_ready"])
|
||||
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
|
||||
self.assertIn("发布闸门:暂不可发布", payload["headline"])
|
||||
self.assertIn("接入缺口:签发接入工单:mainland-controller-01", payload["headline"])
|
||||
self.assertEqual("handover_first_gap", payload["go_live_summary"]["next_step_action_code"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["summary"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(2, payload["summary"]["do_now_total"])
|
||||
self.assertEqual(1, payload["summary"]["standard_path_total"])
|
||||
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_playbook_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertIn("ops_observability_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("priority:fix-handover-gap", payload["top_recommendation"]["key"])
|
||||
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
|
||||
self.assertEqual(2, len(payload["driver_recommendations"]))
|
||||
self.assertEqual(1, len(payload["runbook_sequences"]))
|
||||
self.assertEqual(1, len(payload["activity_focus"]))
|
||||
self.assertEqual(
|
||||
["priority_recommendation", "driver_recommendation", "runbook_sequence"],
|
||||
[item["kind"] for item in payload["entries"]],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("disabled", payload["summary"]["scene_log_status"])
|
||||
self.assertEqual("blocked", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual(2, payload["summary"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["summary"]["backend_handled_total"])
|
||||
self.assertEqual("blocked", payload["automation_coverage"]["launch_status"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["guarded_auto_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["auto_execute_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["resolve_first_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["human_dependency_total"])
|
||||
self.assertFalse(payload["automation_coverage"]["launch_ready"])
|
||||
|
||||
priority_entry = payload["entries"][0]
|
||||
self.assertEqual("do_now", priority_entry["lane"])
|
||||
self.assertEqual("driver_action", priority_entry["executor_kind"])
|
||||
self.assertEqual("handover_first_gap", priority_entry["primary_action_code"])
|
||||
self.assertEqual("ops_driver_contract", priority_entry["contract_navigation"]["primary_contract_key"])
|
||||
|
||||
activity_focus = payload["activity_focus"][0]
|
||||
self.assertEqual("ops_job", activity_focus["activity_kind"])
|
||||
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", activity_focus["summary"])
|
||||
self.assertEqual("focus_latest_job_events", activity_focus["focus_action_code"])
|
||||
self.assertEqual("job_events", activity_focus["ui_intent_kind"])
|
||||
self.assertEqual("ops_job_event", activity_focus["source_focus_ref"]["kind"])
|
||||
self.assertTrue(activity_focus["observation_only"])
|
||||
self.assertEqual("ops_observability_contract", activity_focus["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", activity_focus["contract_navigation"]["contract_keys"])
|
||||
|
||||
scene_log_observation = payload["scene_log_observation"]
|
||||
self.assertEqual("disabled", scene_log_observation["status"])
|
||||
self.assertEqual("mainland-worker-01", scene_log_observation["target_node_code"])
|
||||
self.assertEqual("node_scene_log", scene_log_observation["focus_ref"]["kind"])
|
||||
self.assertEqual(
|
||||
"ops_observability_contract",
|
||||
scene_log_observation["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
|
||||
runbook_entry = payload["entries"][2]
|
||||
self.assertEqual("runbook_sequence", runbook_entry["kind"])
|
||||
self.assertEqual("standard_path", runbook_entry["lane"])
|
||||
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
|
||||
self.assertEqual("release_progression", runbook_entry["sequence_key"])
|
||||
self.assertEqual("fix_managed_nodes", runbook_entry["primary_action_code"])
|
||||
self.assertEqual("focus_release_hub", runbook_entry["secondary_action_code"])
|
||||
self.assertIn("当前解析:补执行面接管", runbook_entry["detail_lines"][0])
|
||||
self.assertEqual("release_hub", runbook_entry["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", runbook_entry["primary_focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", runbook_entry["primary_action_payload"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_action_payload"]["focus_ref"]["section"])
|
||||
self.assertEqual("ops_driver_contract", runbook_entry["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_playbook_contract", runbook_entry["contract_navigation"]["contract_keys"])
|
||||
self.assertIn("release_hub_contract", runbook_entry["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_driver_feed_preserves_bootstrap_run_as_top_recommendation(
|
||||
self,
|
||||
mock_get_ops_overview,
|
||||
mock_get_ops_runbook,
|
||||
mock_get_ops_go_live_summary,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
"publish_ready": False,
|
||||
"publish_status": "attention",
|
||||
"publish_status_label": "可发布但建议先复核",
|
||||
"publish_summary": "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。",
|
||||
"operator_title": "先跑接入收口",
|
||||
"next_step_action_code": "bootstrap_run",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "跑接入收口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"blocking_reasons": [],
|
||||
"warnings": ["remote_agent_ready=1/2"],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"key": "handover-gap-bootstrap_run",
|
||||
"priority": "先补接管缺口",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "Launchpad 已明确收敛到首台节点接入收口。",
|
||||
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "warning",
|
||||
"primary_label": "跑接入收口",
|
||||
"secondary_label": "查看目标节点",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"secondary_action_code": "view_first_gap",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"primary_node_codes": ["mainland-worker-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
"secondary_action_payload": {},
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"driver_recommendations": [],
|
||||
"activity_stream": {"items": []},
|
||||
"execution_scene": {},
|
||||
}
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
payload = get_ops_driver_feed()
|
||||
|
||||
self.assertEqual("priority:handover-gap-bootstrap_run", payload["top_recommendation"]["key"])
|
||||
self.assertEqual("attention", payload["summary"]["publish_status"])
|
||||
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-worker-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("跑接入收口", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual("attention", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("bootstrap_run", payload["top_recommendation"]["primary_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["top_recommendation"]["primary_node_codes"][0])
|
||||
self.assertIn("ops_agent_protocol", payload["top_recommendation"]["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["backend_handled_total"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
24
domain-api/tests/test_ops_service_execution_modes.py
Normal file
24
domain-api/tests/test_ops_service_execution_modes.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import unittest
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_service import _recommend_driver_execution_mode_for_supported_modes
|
||||
|
||||
|
||||
class OpsServiceExecutionModeRecommendationTests(unittest.TestCase):
|
||||
def test_recommend_driver_execution_mode_prefers_local_runtime_for_current_node(self) -> None:
|
||||
result = _recommend_driver_execution_mode_for_supported_modes(
|
||||
[settings.node_code],
|
||||
supported_modes=["remote-agent", "local-runtime", "ssh"],
|
||||
default_mode="remote-agent",
|
||||
)
|
||||
|
||||
self.assertEqual("local-runtime", result["execution_mode"])
|
||||
self.assertIn("local-runtime", result["reason"])
|
||||
self.assertEqual(
|
||||
["remote-agent", "local-runtime", "ssh"],
|
||||
result["supported_modes"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
106
domain-api/tests/test_ops_service_go_live_bundle.py
Normal file
106
domain-api/tests/test_ops_service_go_live_bundle.py
Normal file
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_bundle
|
||||
|
||||
|
||||
class OpsServiceGoLiveBundleTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_bundle_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_bundle_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
|
||||
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("missing", payload["status"])
|
||||
self.assertFalse(payload["bundle_available"])
|
||||
self.assertEqual("待导出", payload["status_label"])
|
||||
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_bundle_reads_manifest_summary_and_review_state(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_bundle_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
env_audit_path = report_dir / "00_env_audit.txt"
|
||||
env_audit_path.write_text(
|
||||
"[8/8] condensed env audit summary\n"
|
||||
+ json.dumps(
|
||||
{
|
||||
"status": "attention",
|
||||
"headline": "仍有轻微环境缺口",
|
||||
"missing_items": ["missing_tool:jq"],
|
||||
"runtime": {"runtime_may_need_restart": True},
|
||||
"recommended_actions": ["bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
|
||||
{"key": "go_live_summary", "path": str(report_dir / "02_go_live_summary.json"), "ok": True},
|
||||
],
|
||||
"failures": ["doctor_export"],
|
||||
"summary": {
|
||||
"ok": False,
|
||||
"artifact_total": 9,
|
||||
"failure_total": 1,
|
||||
"review_status_hint": "attention",
|
||||
"review_headline": "核心报告已齐,但建议先复核补充缺口。",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接管缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"launchpad_alignment": {"consistent": False, "target_node_code_consistent": False},
|
||||
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
|
||||
"contract_surface_gaps": ["ops_go_live_bundle_contract"],
|
||||
"discovered_contract_keys": ["ops_go_live_signoff_contract"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertTrue(payload["bundle_available"])
|
||||
self.assertEqual(str(manifest_path), payload["manifest_path"])
|
||||
self.assertEqual(9, payload["artifact_total"])
|
||||
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
|
||||
self.assertEqual("attention", payload["env_audit"]["status"])
|
||||
self.assertTrue(payload["env_audit"]["runtime_may_need_restart"])
|
||||
self.assertFalse(payload["summary"]["launchpad_alignment"]["consistent"])
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_signoff"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
169
domain-api/tests/test_ops_service_go_live_review.py
Normal file
169
domain-api/tests/test_ops_service_go_live_review.py
Normal file
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_review
|
||||
|
||||
|
||||
class OpsServiceGoLiveReviewTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_review_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_review_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
|
||||
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("missing", payload["status"])
|
||||
self.assertEqual("待导出", payload["status_label"])
|
||||
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_review_reads_manifest_and_alignment(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_review_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
env_audit_path = report_dir / "00_env_audit.txt"
|
||||
env_audit_path.write_text(
|
||||
"[8/8] condensed env audit summary\n"
|
||||
+ json.dumps(
|
||||
{
|
||||
"status": "attention",
|
||||
"headline": "环境仍有轻微缺口",
|
||||
"missing_items": ["missing_tool:jq"],
|
||||
"recommended_actions": ["bash runtime-refresh-recover"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
go_live_summary_path = report_dir / "02_go_live_summary.json"
|
||||
go_live_summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"go_live_status": "attention",
|
||||
"publish_status": "attention",
|
||||
"operator_title": "先补接入缺口",
|
||||
"next_step_action_code": "fix_managed_nodes",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stack_diagnosis_path = report_dir / "03_stack_diagnosis.json"
|
||||
stack_diagnosis_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"diagnosis": {
|
||||
"stack_status": "attention",
|
||||
"operator_decision": {"title": "先补接入缺口"},
|
||||
"next_step": {"action_code": "fix_managed_nodes"},
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-02",
|
||||
"launchpad_recommended_recovery_label": "先看节点接管",
|
||||
"launchpad_recommended_recovery_summary": "stack 与 summary 存在漂移。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
driver_feed_path = report_dir / "04_driver_feed.json"
|
||||
driver_feed_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"headline": "driver ok",
|
||||
"summary": {
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
codex_brief_path = report_dir / "05_codex_brief.json"
|
||||
codex_brief_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"headline": "codex ok",
|
||||
"summary": {
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
|
||||
"focus": {"entry_key": "fix_managed_nodes"},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
|
||||
{"key": "go_live_summary", "path": str(go_live_summary_path), "ok": True},
|
||||
{"key": "stack_diagnosis", "path": str(stack_diagnosis_path), "ok": True},
|
||||
{"key": "driver_feed", "path": str(driver_feed_path), "ok": True},
|
||||
{"key": "codex_brief", "path": str(codex_brief_path), "ok": True},
|
||||
{"key": "release_launchpad", "path": str(report_dir / "06_release_launchpad.json"), "ok": True},
|
||||
],
|
||||
"failures": ["doctor_export"],
|
||||
"summary": {
|
||||
"artifact_total": 9,
|
||||
"failure_total": 1,
|
||||
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertEqual(9, payload["artifact_total"])
|
||||
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertIn("go_live_summary", payload["launchpad_alignment"]["available_sources"])
|
||||
self.assertIn("mainland-worker-01", payload["recommended_next_steps"][0] + "".join(payload["recommended_next_steps"]))
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_review"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
317
domain-api/tests/test_ops_service_go_live_signoff.py
Normal file
317
domain-api/tests/test_ops_service_go_live_signoff.py
Normal file
@@ -0,0 +1,317 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_signoff
|
||||
|
||||
|
||||
class OpsServiceGoLiveSignoffTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_ready_when_all_surfaces_align(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "ready",
|
||||
"publish_status": "ready",
|
||||
"operator_title": "进入发布",
|
||||
"next_step_action_code": "publish_latest_worker",
|
||||
"next_step_reason": "当前门禁已收口。",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {"next_step": "bash next-step"},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"stack_status": "ready",
|
||||
"operator_decision": {"title": "进入发布"},
|
||||
"next_step": {"action_code": "publish_latest_worker"},
|
||||
}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "主车道清晰",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "可签字",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {"entry_key": "publish_latest_worker"},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "继续发布",
|
||||
"recommended_recovery_summary": "可以进入正式发布。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "ready",
|
||||
"status_label": "可复核发布",
|
||||
"headline": "review ok",
|
||||
"report_dir": "/tmp/review",
|
||||
"manifest_path": "/tmp/review/manifest.json",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "doctor ok",
|
||||
"report_dir": "/tmp/review",
|
||||
"manifest_path": "/tmp/review/manifest.json",
|
||||
"decision": {
|
||||
"reason_code": "all_green",
|
||||
"preferred_surface": "go-live-summary",
|
||||
"next_action_code": "publish_latest_worker",
|
||||
},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("ready", payload["signoff_status"])
|
||||
self.assertEqual("可签字上线", payload["status_label"])
|
||||
self.assertEqual([], payload["blocked_reasons"])
|
||||
self.assertEqual([], payload["attention_reasons"])
|
||||
self.assertTrue(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertEqual("go-live-summary", payload["decision"]["preferred_surface"])
|
||||
self.assertIn("go-live-signoff", payload["recommended_commands"]["go_live_signoff"])
|
||||
self.assertEqual("ready", payload["release_gate"]["go_live_review_status"])
|
||||
self.assertEqual("ready", payload["release_gate"]["doctor_decision_status"])
|
||||
self.assertEqual("/tmp/review/manifest.json", payload["manifest_path"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_attention_when_launchpad_alignment_drifts(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "ready",
|
||||
"publish_status": "ready",
|
||||
"operator_title": "进入发布",
|
||||
"next_step_action_code": "publish_latest_worker",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {"stack_status": "ready", "operator_decision": {}, "next_step": {}}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "主车道清晰",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-02",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "可签字",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "继续发布",
|
||||
"recommended_recovery_summary": "可以进入正式发布。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "review drift",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "doctor ok",
|
||||
"decision": {"preferred_surface": "go-live-summary"},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff()
|
||||
|
||||
self.assertEqual("attention", payload["signoff_status"])
|
||||
self.assertIn("launchpad_alignment=inconsistent", payload["attention_reasons"])
|
||||
self.assertIn("go_live_review=attention", payload["attention_reasons"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["target_node_code_consistent"])
|
||||
self.assertEqual("go-live-review", payload["decision"]["preferred_surface"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_blocked_when_publish_gate_is_blocked(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
"publish_status": "blocked",
|
||||
"operator_title": "先处理发布阻断",
|
||||
"next_step_action_code": "fix_rollout_blockers",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"stack_status": "attention",
|
||||
"operator_decision": {"title": "先看发布门禁"},
|
||||
"next_step": {"action_code": "fix_rollout_blockers"},
|
||||
}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "发布门禁阻断",
|
||||
"summary": {
|
||||
"launch_status": "attention",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "需先排阻断",
|
||||
"summary": {
|
||||
"launch_status": "attention",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {"entry_key": "fix_rollout_blockers"},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "blocked",
|
||||
"status_label": "发布阻断",
|
||||
"recommended_action_code": "fix_rollout_blockers",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "处理发布阻断",
|
||||
"recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "review ok",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "doctor focus",
|
||||
"decision": {
|
||||
"reason_code": "publish_blocked",
|
||||
"preferred_surface": "release-launchpad",
|
||||
"next_action_code": "fix_rollout_blockers",
|
||||
},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff()
|
||||
|
||||
self.assertEqual("blocked", payload["signoff_status"])
|
||||
self.assertIn("publish_status=blocked", payload["blocked_reasons"])
|
||||
self.assertIn("release_launchpad=blocked", payload["blocked_reasons"])
|
||||
self.assertEqual("release-launchpad", payload["decision"]["preferred_surface"])
|
||||
self.assertEqual("publish_blocked", payload["decision"]["doctor_reason_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
664
domain-api/tests/test_ops_service_go_live_summary.py
Normal file
664
domain-api/tests/test_ops_service_go_live_summary.py
Normal file
@@ -0,0 +1,664 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_summary
|
||||
|
||||
|
||||
class OpsServiceGoLiveSummaryTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "attention",
|
||||
"next_step": {
|
||||
"action_code": "api-restart",
|
||||
"reason": "build schema stale",
|
||||
"focus_ref": {"kind": "runtime_build_info", "section": "schema"},
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "runtime_recovery",
|
||||
"title": "优先重启控制面 API 并复检运行时版本",
|
||||
"primary_command_key": "api-restart",
|
||||
},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"repository_capabilities": {
|
||||
"supports_install_command_block": True,
|
||||
"supports_multi_layout_bootstrap": True,
|
||||
},
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {
|
||||
"ops_contracts": "/api/v1/ops/contracts",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先重启 API",
|
||||
"reason": "build schema stale",
|
||||
"primary_action_code": "api-restart",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"participating_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertEqual("attention", payload["publish_status"])
|
||||
self.assertTrue(payload["runtime_schema_stale"])
|
||||
self.assertFalse(payload["route_surface_declares_bootstrap_plan"])
|
||||
self.assertIn("runtime_build_schema_stale", payload["warnings"])
|
||||
self.assertIn("runtime_build_schema_stale", payload["publish_warnings"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["go_live_recover"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["api_restart"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["runtime_refresh_recover"],
|
||||
)
|
||||
self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_highlights_remote_access_and_log_sync_gaps(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "blocked",
|
||||
"next_step": {
|
||||
"action_code": "handover_first_gap",
|
||||
"reason": "来自 overview.recommendation.primary_action_code",
|
||||
},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"focus_ref": {"kind": "release_hub"},
|
||||
"onboarding_bootstrap_pending_nodes": 1,
|
||||
"onboarding_acceptance_ready_nodes": 1,
|
||||
"recommended_target_node_code": "mainland-controller-01",
|
||||
"recommended_recovery_label": "签发接入工单",
|
||||
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 3,
|
||||
"remote_access_ready": 0,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补接管缺口",
|
||||
"reason": "当前仍有节点没有进入标准执行器就绪状态。",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "waiting_sample",
|
||||
"mode": "full",
|
||||
"participating_node_count": 3,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [
|
||||
"mainland-worker-01",
|
||||
"mainland-controller-01",
|
||||
"overseas-control-01",
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("blocked", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("blocked", payload["publish_status"])
|
||||
self.assertEqual("暂不可发布", payload["publish_status_label"])
|
||||
self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"])
|
||||
self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"])
|
||||
self.assertIn("remote_agent_ready=0", payload["blocking_reasons"])
|
||||
self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"])
|
||||
self.assertIn("release_launchpad=attention", payload["warnings"])
|
||||
self.assertIn("release_launchpad=attention", payload["publish_warnings"])
|
||||
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"])
|
||||
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"])
|
||||
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"])
|
||||
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"])
|
||||
self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"])
|
||||
self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"])
|
||||
self.assertTrue(payload["route_surface_complete"])
|
||||
self.assertEqual("先补接管缺口", payload["operator_title"])
|
||||
self.assertEqual("handover_first_gap", payload["next_step_action_code"])
|
||||
self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating",
|
||||
payload["recommended_commands"]["log_sync_logs"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "blocked",
|
||||
"next_step": {
|
||||
"action_code": "bootstrap_run",
|
||||
"reason": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "node_handover",
|
||||
"title": "优先执行节点接管收口",
|
||||
"primary_command_key": "bootstrap_run",
|
||||
},
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {},
|
||||
},
|
||||
"repository_capabilities": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补接管缺口",
|
||||
"reason": "overview 仍在泛化描述。",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"participating_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("bootstrap_run", payload["next_step_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
|
||||
self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {
|
||||
"action_code": "run_inspection_participating",
|
||||
"reason": "继续做上线前复核。",
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "go-live",
|
||||
"title": "可以进入上线复核",
|
||||
"primary_command_key": "run_inspection_participating",
|
||||
},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 2,
|
||||
"remote_access_ready": 2,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "继续复核",
|
||||
"reason": "当前执行面与观测面已收口。",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("ready", payload["go_live_status"])
|
||||
self.assertTrue(payload["publish_ready"])
|
||||
self.assertEqual("ready", payload["publish_status"])
|
||||
self.assertEqual("可发布", payload["publish_status_label"])
|
||||
self.assertEqual([], payload["blocking_reasons"])
|
||||
self.assertEqual([], payload["warnings"])
|
||||
self.assertEqual([], payload["publish_blocking_reasons"])
|
||||
self.assertEqual([], payload["publish_warnings"])
|
||||
self.assertEqual("go-live", payload["operator_lane"])
|
||||
self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"])
|
||||
self.assertEqual(2, payload["log_sync_covered_nodes"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 2,
|
||||
"remote_access_ready": 2,
|
||||
"queue_dead_letter_nodes": 1,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先清死信",
|
||||
"reason": "当前仍有节点回执死信。",
|
||||
"primary_action_code": "delivery_queue_replay",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("blocked", payload["publish_status"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "attention",
|
||||
"next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 3,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补大陆接管",
|
||||
"reason": "当前只有海外样本已形成。",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "partial_coverage",
|
||||
"mode": "full",
|
||||
"participating_node_count": 3,
|
||||
"covered_participating_node_count": 1,
|
||||
"covered_participating_nodes": ["overseas-control-01"],
|
||||
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertEqual(1, payload["log_sync_covered_nodes"])
|
||||
self.assertIn("log_sync_partial=1/3", payload["warnings"])
|
||||
self.assertIn("log_sync_partial=1/3", payload["publish_warnings"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_marks_attention_for_start_delivery_gap(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {
|
||||
"action_code": "focus_latest_job_events",
|
||||
"reason": "先看任务事件。",
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "ops_jobs",
|
||||
"title": "优先检查任务事件流",
|
||||
"primary_command_key": "focus_latest_job_events",
|
||||
},
|
||||
"issues": [
|
||||
{
|
||||
"code": "ops_job_start_delivery_failed_local",
|
||||
"summary": "最近有任务开始回执异常。",
|
||||
}
|
||||
],
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先看任务事件",
|
||||
"reason": "最近出现开始回执异常。",
|
||||
"primary_action_code": "focus_latest_job_events",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 1,
|
||||
"covered_participating_node_count": 1,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("attention", payload["publish_status"])
|
||||
self.assertEqual(1, payload["activity_start_delivery_issue_total"])
|
||||
self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"])
|
||||
self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"])
|
||||
self.assertEqual("ops_jobs", payload["operator_lane"])
|
||||
self.assertEqual("focus_latest_job_events", payload["next_step_action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
182
domain-api/tests/test_ops_service_link_snapshot.py
Normal file
182
domain-api/tests/test_ops_service_link_snapshot.py
Normal file
@@ -0,0 +1,182 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_link_snapshot
|
||||
|
||||
|
||||
class OpsServiceLinkSnapshotTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_link_snapshot_highlights_log_sync_gap_on_active_scene(self, mock_get_ops_overview) -> None:
|
||||
mock_get_ops_overview.return_value = {
|
||||
"current_topology": {
|
||||
"node_code": "overseas-control-01",
|
||||
"node_region": "overseas",
|
||||
"node_role": "control",
|
||||
"worker_mode": "linux-systemd",
|
||||
"sync_push_enabled": True,
|
||||
"sync_target_api_base_url": "http://152.53.37.118:8100",
|
||||
},
|
||||
"cluster_summary": {
|
||||
"online_control_nodes": 2,
|
||||
"online_worker_nodes": 3,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"busy_nodes": ["mainland-worker-01", "overseas-control-01"],
|
||||
},
|
||||
"runtime_summary": {
|
||||
"readiness_status": "ready",
|
||||
"readiness_summary": "当前多机与跨地域骨架已进入可联调状态。",
|
||||
"phase_label": "running",
|
||||
"phase_detail": "当前任务正在推进",
|
||||
"active_job_code": "detect-20260417164852-75f7cc",
|
||||
"progress_percent": 42.5,
|
||||
},
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"effective_online_nodes": 3,
|
||||
"participating_nodes": 3,
|
||||
"dispatch_active_nodes": 2,
|
||||
"recent_only_nodes": 1,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "overseas-control-01"],
|
||||
"non_participating_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 3,
|
||||
"missing_participating_nodes": [
|
||||
"mainland-controller-01",
|
||||
"mainland-worker-01",
|
||||
"overseas-control-01",
|
||||
],
|
||||
"last_at": "",
|
||||
"preview_lines": [],
|
||||
},
|
||||
},
|
||||
"sync_summary": {
|
||||
"enabled": True,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"projected_batches": 0,
|
||||
"failed_batches": 0,
|
||||
"records_total": 2513,
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "enable-scene-log-sync",
|
||||
"priority": "先补现场日志回传",
|
||||
"summary": "当前有 3 台节点正在真实参与检测,但海外控制面还看不到过程日志。",
|
||||
"reason": "先把远端日志回传切到关键模式,才能看到阶段、异常、代理与执行过程。",
|
||||
"primary_label": "开启关键回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
|
||||
"primary_action_payload": {},
|
||||
"secondary_label": "开启全量回传",
|
||||
"secondary_action_code": "enable_log_sync_full",
|
||||
"secondary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
}
|
||||
|
||||
snapshot = get_ops_link_snapshot()
|
||||
|
||||
self.assertEqual("attention", snapshot["status"])
|
||||
self.assertIn("日志回传", snapshot["headline"])
|
||||
self.assertIn("活跃任务 detect-20260417164852-75f7cc", snapshot["summary_lines"][2])
|
||||
self.assertIn("远端日志:关闭", snapshot["summary_lines"][4])
|
||||
self.assertEqual("enable_log_sync_key", snapshot["next_actions"][0]["action_code"])
|
||||
self.assertEqual(3, snapshot["execution"]["participating_nodes"])
|
||||
self.assertFalse(snapshot["log_sync"]["enabled"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_link_snapshot_reports_ready_when_scene_is_observable(self, mock_get_ops_overview) -> None:
|
||||
mock_get_ops_overview.return_value = {
|
||||
"current_topology": {
|
||||
"node_code": "overseas-control-01",
|
||||
"node_region": "overseas",
|
||||
"node_role": "control",
|
||||
"worker_mode": "linux-systemd",
|
||||
"sync_push_enabled": True,
|
||||
"sync_target_api_base_url": "http://152.53.37.118:8100",
|
||||
},
|
||||
"cluster_summary": {
|
||||
"online_control_nodes": 2,
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"busy_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
"runtime_summary": {
|
||||
"readiness_status": "ready",
|
||||
"readiness_summary": "ok",
|
||||
"phase_label": "running",
|
||||
"phase_detail": "检测任务正常推进",
|
||||
"active_job_code": "detect-001",
|
||||
"progress_percent": 66.0,
|
||||
},
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"effective_online_nodes": 2,
|
||||
"participating_nodes": 2,
|
||||
"dispatch_active_nodes": 2,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"non_participating_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 18,
|
||||
"source_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
"last_at": "2026-04-17 22:20:00",
|
||||
"preview_lines": [
|
||||
"[2026-04-17 22:19:59] [mainland-worker-01] 开始检测域名: alpha.com",
|
||||
"[2026-04-17 22:20:00] [mainland-controller-01] 域名检测完成: beta.com",
|
||||
],
|
||||
},
|
||||
},
|
||||
"sync_summary": {
|
||||
"enabled": True,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"projected_batches": 0,
|
||||
"failed_batches": 0,
|
||||
"records_total": 2600,
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "inspect-participating",
|
||||
"priority": "先看执行现场",
|
||||
"summary": "当前有 2 台节点正在真实参与检测,优先回收它们的标准巡检结果。",
|
||||
"reason": "真正参与检测的节点最接近现场。",
|
||||
"primary_label": "执行标准巡检",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_action_payload": {},
|
||||
"secondary_label": "打开诊断模板",
|
||||
"secondary_action_code": "open_diagnostics_participating",
|
||||
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
}
|
||||
|
||||
snapshot = get_ops_link_snapshot()
|
||||
|
||||
self.assertEqual("ready", snapshot["status"])
|
||||
self.assertIn("可观测状态", snapshot["headline"])
|
||||
self.assertIn("覆盖 2/2", snapshot["summary_lines"][4])
|
||||
self.assertEqual(2, snapshot["log_sync"]["source_node_count"])
|
||||
self.assertEqual(2, len(snapshot["next_actions"]))
|
||||
self.assertEqual("run_inspection_participating", snapshot["next_actions"][0]["action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
256
domain-api/tests/test_ops_service_node_scene_log.py
Normal file
256
domain-api/tests/test_ops_service_node_scene_log.py
Normal file
@@ -0,0 +1,256 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_node_scene_log
|
||||
|
||||
|
||||
class OpsServiceNodeSceneLogTests(unittest.TestCase):
|
||||
def _runtime_payload_with_worker_logs(self) -> dict:
|
||||
return {
|
||||
"detect": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 2,
|
||||
"source_node_count": 1,
|
||||
"source_nodes": ["mainland-worker-01"],
|
||||
"source_node_summaries": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"line_count": 2,
|
||||
"key_line_count": 2,
|
||||
"full_line_count": 0,
|
||||
"last_at": "2026-04-18 05:02:00",
|
||||
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
}
|
||||
],
|
||||
"preview_lines": [
|
||||
"[2026-04-18 05:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
|
||||
"[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
],
|
||||
"last_at": "2026-04-18 05:02:00",
|
||||
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
},
|
||||
"participation_summary": {
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 1,
|
||||
"standby_nodes": 1,
|
||||
"load_syncing_nodes": 0,
|
||||
"effective_online_nodes": 2,
|
||||
"dedicated_worker_nodes": 2,
|
||||
"controller_worker_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
"non_participating_node_codes": ["mainland-worker-02"],
|
||||
"standby_node_codes": ["mainland-worker-02"],
|
||||
"load_syncing_node_codes": [],
|
||||
"summary": "有效执行节点 2 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 1 台",
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 2,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:10",
|
||||
"participation_state": "claimed",
|
||||
"participation_label": "已领待跑",
|
||||
"participation_reason": "已领取 2 项任务,等待线程继续执行。",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:11",
|
||||
"participation_state": "standby",
|
||||
"participation_label": "在线待命",
|
||||
"participation_reason": "当前节点在线,但尚未参与本轮检测。",
|
||||
"is_dispatch_active": False,
|
||||
}
|
||||
],
|
||||
"active_job": {
|
||||
"job_code": "detect-001",
|
||||
"status": "running",
|
||||
"current_cycle_token": "cycle-1",
|
||||
"current_cycle_events": [
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:02:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "域名检测完成: alpha.com",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:01:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "代理池刷新完成",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "full"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:00:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "开始检测域名: alpha.com",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 04:59:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "旧周期日志",
|
||||
"payload": {"cycle_token": "cycle-0", "log_mode": "key"},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
"cluster": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 2,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:10",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:11",
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_filters_key_mode_records_for_participating_node(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-01", limit=10)
|
||||
|
||||
self.assertEqual("mainland-worker-01", payload["node_code"])
|
||||
self.assertEqual("healthy", payload["status"])
|
||||
self.assertEqual("关键覆盖", payload["status_label"])
|
||||
self.assertEqual("key", payload["mode"])
|
||||
self.assertTrue(payload["log_sync_enabled"])
|
||||
self.assertTrue(payload["participation"]["detect_participating"])
|
||||
self.assertEqual(2, payload["records_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"开始检测域名: alpha.com",
|
||||
"域名检测完成: alpha.com",
|
||||
],
|
||||
[item["message"] for item in payload["records"]],
|
||||
)
|
||||
self.assertEqual(2, payload["source_summary"]["line_count"])
|
||||
self.assertEqual(2, payload["source_summary"]["key_line_count"])
|
||||
self.assertEqual(0, payload["source_summary"]["full_line_count"])
|
||||
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_can_request_full_mode_records(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-01", limit=10, mode="full")
|
||||
|
||||
self.assertEqual("full_capture", payload["status"])
|
||||
self.assertEqual("full", payload["mode"])
|
||||
self.assertEqual(3, payload["records_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"开始检测域名: alpha.com",
|
||||
"代理池刷新完成",
|
||||
"域名检测完成: alpha.com",
|
||||
],
|
||||
[item["message"] for item in payload["records"]],
|
||||
)
|
||||
self.assertEqual(3, payload["source_summary"]["line_count"])
|
||||
self.assertEqual(2, payload["source_summary"]["key_line_count"])
|
||||
self.assertEqual(1, payload["source_summary"]["full_line_count"])
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_marks_waiting_sample_for_missing_participating_node(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"detect": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"source_nodes": [],
|
||||
"source_node_summaries": [],
|
||||
"preview_lines": [],
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
},
|
||||
"participation_summary": {
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"effective_online_nodes": 1,
|
||||
"dedicated_worker_nodes": 1,
|
||||
"controller_worker_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-02"],
|
||||
"recent_only_node_codes": [],
|
||||
"non_participating_node_codes": [],
|
||||
"standby_node_codes": [],
|
||||
"load_syncing_node_codes": [],
|
||||
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 1,
|
||||
"last_heartbeat_at": "2026-04-18 05:05:00",
|
||||
"participation_state": "running",
|
||||
"participation_label": "正在执行",
|
||||
"participation_reason": "当前存在执行中工作负载。",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [],
|
||||
"active_job": {"job_code": "detect-002", "status": "running", "current_cycle_events": []},
|
||||
},
|
||||
"cluster": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 1,
|
||||
"last_heartbeat_at": "2026-04-18 05:05:00",
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-02")
|
||||
|
||||
self.assertEqual("waiting_sample", payload["status"])
|
||||
self.assertEqual("等待样本", payload["status_label"])
|
||||
self.assertEqual(0, payload["records_total"])
|
||||
self.assertTrue(payload["participation"]["detect_participating"])
|
||||
self.assertIn("现场样本尚未形成", payload["summary"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1186
domain-api/tests/test_ops_service_overview.py
Normal file
1186
domain-api/tests/test_ops_service_overview.py
Normal file
File diff suppressed because it is too large
Load Diff
283
domain-api/tests/test_ops_service_runbook.py
Normal file
283
domain-api/tests/test_ops_service_runbook.py
Normal file
@@ -0,0 +1,283 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_runbook
|
||||
|
||||
|
||||
class OpsServiceRunbookTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_includes_control_sequences(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "ready",
|
||||
"summary": "当前多机现场已进入标准作业状态",
|
||||
},
|
||||
"worker": {
|
||||
"running": True,
|
||||
},
|
||||
"sync_agent": {
|
||||
"running": True,
|
||||
},
|
||||
"detect": {
|
||||
"participation_summary": {
|
||||
"summary": "worker 正在执行",
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"role": "worker",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "busy",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.side_effect = lambda channel="stable": (
|
||||
{
|
||||
"id": 15,
|
||||
"release_version": "2026.04.17",
|
||||
"channel": "stable",
|
||||
}
|
||||
if channel == "stable"
|
||||
else {}
|
||||
)
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"channel": "stable",
|
||||
"latest_release": {
|
||||
"id": 15,
|
||||
"release_version": "2026.04.17",
|
||||
"channel": "stable",
|
||||
},
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。",
|
||||
"recommended_action_code": "publish_latest_worker",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 15,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"message": "智能 Rollout 预检通过",
|
||||
"smart_rollout": {
|
||||
"available": True,
|
||||
"summary": "已按 Worker 模式预选 1 台节点。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"message": "",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
|
||||
mock_list_managed_nodes_with_agent_state.assert_called_once_with(
|
||||
participation_payload=mock_get_runtime_status.return_value["detect"]
|
||||
)
|
||||
self.assertIn("GET /api/v1/ops/releases/launchpad", runbook["fast_checks"])
|
||||
self.assertEqual("ready", runbook["release_launchpad"]["launchpad_status"]["status"])
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
self.assertIn("node_onboarding", sequences)
|
||||
self.assertIn("scene_observe", sequences)
|
||||
self.assertIn("standard_inspection", sequences)
|
||||
self.assertIn("release_progression", sequences)
|
||||
self.assertEqual("bootstrap_run", sequences["node_onboarding"]["primary_action_code"])
|
||||
self.assertEqual("run_acceptance", sequences["node_onboarding"]["secondary_action_code"])
|
||||
self.assertEqual("run_standard_inspection", sequences["standard_inspection"]["primary_action_code"])
|
||||
self.assertEqual("open_diagnostics", sequences["standard_inspection"]["secondary_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["scene_observe"]["target_node_codes"])
|
||||
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual(15, sequences["release_progression"]["primary_action_payload"]["release_id"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["primary_resolution"]["driver_node_codes"])
|
||||
self.assertEqual("release_hub", sequences["release_progression"]["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", sequences["release_progression"]["focus_ref"]["section"])
|
||||
self.assertEqual("worker", sequences["release_progression"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", sequences["release_progression"]["secondary_focus_ref"]["section"])
|
||||
self.assertTrue(sequences["scene_observe"]["secondary_resolution"]["ok"])
|
||||
self.assertEqual("run_scene_logs_full", sequences["scene_observe"]["secondary_resolution"]["driver_action_code"])
|
||||
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_release_progression_can_fall_back_to_bootstrap_run(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "attention",
|
||||
"summary": "当前还需补执行面接管",
|
||||
},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {"participation_summary": {"summary": ""}},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"role": "worker",
|
||||
"is_managed": False,
|
||||
"is_enabled": False,
|
||||
"is_agent_online": False,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.return_value = {}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"channel": "stable",
|
||||
"latest_release": {},
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 19,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"message": "当前没有满足角色条件的在线节点可用于智能 Rollout。",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
"policy_preview": {
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"is_managed": False,
|
||||
"is_enabled": False,
|
||||
"is_agent_online": False,
|
||||
"remote_agent_ready": False,
|
||||
"inspection_status": "missing",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"agent_reason": "节点尚未纳管,当前不能稳定下发 remote-agent 动作。",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"message": "",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
|
||||
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual(["mainland-worker-02"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertEqual("待补执行面", sequences["release_progression"]["status_label"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_release_progression_falls_back_to_direct_worker_rollout_when_release_exists(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "ready",
|
||||
"summary": "当前多机现场已进入标准作业状态",
|
||||
},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {"participation_summary": {"summary": ""}},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"role": "worker",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.return_value = {
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
|
||||
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual("focus_release_hub", sequences["release_progression"]["secondary_action_code"])
|
||||
self.assertEqual(18, sequences["release_progression"]["primary_action_payload"]["release_id"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
255
domain-api/tests/test_ops_service_runbook_sequence.py
Normal file
255
domain-api/tests/test_ops_service_runbook_sequence.py
Normal file
@@ -0,0 +1,255 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_ops_runbook_sequence, resolve_ops_runbook_sequence
|
||||
|
||||
|
||||
class OpsServiceRunbookSequenceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_resolve_ops_runbook_sequence_uses_primary_action(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "可以直接发布。",
|
||||
"reason": "launchpad 已就绪",
|
||||
"target_scope_label": "Release / Launchpad / 2026.04.18",
|
||||
"step_titles": ["确认发布包", "发灰度"],
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 12,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "直接发 Worker 灰度",
|
||||
"primary_action_code": "publish_latest_worker",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 12,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = resolve_ops_runbook_sequence(
|
||||
"release_progression",
|
||||
{
|
||||
"requested_by": "test",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("标准作业路径解析成功", message)
|
||||
self.assertEqual("publish_latest_worker", data["driver_action_code"])
|
||||
self.assertEqual("直接发 Worker 灰度", data["driver_action_label"])
|
||||
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
|
||||
self.assertEqual("test/runbook/release_progression", data["driver_requested_by"])
|
||||
self.assertEqual("Release / Launchpad / 2026.04.18", data["sequence"]["target_scope_label"])
|
||||
self.assertEqual(["确认发布包", "发灰度"], data["sequence"]["step_titles"])
|
||||
self.assertEqual("release_launchpad", data["sequence"]["focus_ref"]["section"])
|
||||
self.assertEqual("worker", data["sequence"]["action_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["driver_action_payload"]["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_resolve_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "node_onboarding",
|
||||
"title": "纳管新节点",
|
||||
"status": "warning",
|
||||
"status_label": "有待接管节点",
|
||||
"summary": "先补接入。",
|
||||
"reason": "新增节点需要先纳管。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"secondary_label": "跑接管验收",
|
||||
"secondary_action_code": "run_acceptance",
|
||||
"secondary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = resolve_ops_runbook_sequence(
|
||||
"node_onboarding",
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "test",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"auto_approve": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("标准作业路径解析成功", message)
|
||||
self.assertEqual("run_acceptance", data["driver_action_code"])
|
||||
self.assertEqual("跑接管验收", data["driver_action_label"])
|
||||
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
|
||||
self.assertTrue(data["sequence"]["secondary"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"auto_approve": True,
|
||||
},
|
||||
data["driver_action_payload"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_uses_primary_action(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "可以直接发布。",
|
||||
"reason": "launchpad 已就绪",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_code": "publish_latest_worker",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"secondary_action_payload": {
|
||||
"release_id": 12,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "smart-release-rollout",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence(
|
||||
"release_progression",
|
||||
{
|
||||
"requested_by": "test",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["handled"])
|
||||
self.assertEqual("publish_latest_worker", data["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
|
||||
self.assertEqual("release_progression", data["sequence_key"])
|
||||
self.assertEqual("版本发布与放量", data["sequence"]["title"])
|
||||
mock_execute_driver_action.assert_called_once_with(
|
||||
{
|
||||
"action_code": "publish_latest_worker",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"requested_by": "test/runbook/release_progression",
|
||||
}
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "node_onboarding",
|
||||
"title": "纳管新节点",
|
||||
"status": "warning",
|
||||
"status_label": "有待接管节点",
|
||||
"summary": "先补接入。",
|
||||
"reason": "新增节点需要先纳管。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"primary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"secondary_action_code": "run_acceptance",
|
||||
"secondary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "ui-intent",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence(
|
||||
"node_onboarding",
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "test",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"auto_approve": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("", message)
|
||||
self.assertTrue(data["handled"])
|
||||
self.assertEqual("run_acceptance", data["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
|
||||
self.assertTrue(data["sequence"]["secondary"])
|
||||
mock_execute_driver_action.assert_called_once_with(
|
||||
{
|
||||
"action_code": "run_acceptance",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
"auto_approve": True,
|
||||
},
|
||||
"requested_by": "test/runbook/node_onboarding",
|
||||
}
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_returns_error_when_sequence_missing(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence("missing-sequence", {})
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("标准作业路径不存在", message)
|
||||
self.assertEqual("missing-sequence", data["sequence_key"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
971
domain-api/tests/test_ops_service_stack_diagnosis.py
Normal file
971
domain-api/tests/test_ops_service_stack_diagnosis.py
Normal file
@@ -0,0 +1,971 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_stack_diagnosis
|
||||
|
||||
|
||||
class OpsServiceStackDiagnosisTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"title": "先补节点接管",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"source": "git",
|
||||
"commit_sha": "246838ae4c07",
|
||||
"repository_capabilities": {
|
||||
"supports_install_command_block": True,
|
||||
"supports_multi_layout_bootstrap": True,
|
||||
},
|
||||
"route_surface": {
|
||||
"mode": "registered",
|
||||
"registered_paths_total": 24,
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {
|
||||
"ops_contracts": "/api/v1/ops/contracts",
|
||||
"ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
|
||||
self.assertEqual("api-restart", issue["action_code"])
|
||||
self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
|
||||
self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
|
||||
self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["recommended_commands"]["api-restart"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
|
||||
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
|
||||
)
|
||||
self.assertEqual(
|
||||
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
|
||||
payload["diagnosis"]["recommended_commands"]["stack-check"],
|
||||
)
|
||||
self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
|
||||
self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
|
||||
self.assertTrue(
|
||||
any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"source": "git",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"route_surface": {
|
||||
"mode": "registered",
|
||||
"registered_paths_total": 12,
|
||||
"surface_complete": False,
|
||||
"missing_keys": [
|
||||
"ops_node_onboarding_acceptance_execute",
|
||||
"ops_node_onboarding_acceptance_preview",
|
||||
],
|
||||
"missing_paths": [
|
||||
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
|
||||
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
|
||||
)
|
||||
self.assertEqual("api-restart", issue["action_code"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
|
||||
self.assertIn(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
issue["commands"],
|
||||
)
|
||||
self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
|
||||
self.assertIn(
|
||||
"ops_node_onboarding_acceptance_preview",
|
||||
payload["runtime_build_info"]["route_surface_missing_keys"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "参与节点样本尚未形成。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {
|
||||
"participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
|
||||
"participation_summary": {
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 1,
|
||||
"missing_participating_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("observability", decision["lane"])
|
||||
self.assertEqual("scene-node-log", decision["primary_command_key"])
|
||||
self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
|
||||
self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "存在接管缺口。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "ssh_ready",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "等待执行 bootstrap。",
|
||||
"stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("node_handover", decision["lane"])
|
||||
self.assertEqual("bootstrap_run", decision["primary_command_key"])
|
||||
self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
|
||||
self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("当前第一动作已经收敛为 接入收口:mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"title": "先补接管缺口",
|
||||
"reason": "overview 只能先定位到接管缺口。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "agent_pending",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
|
||||
"stage": {
|
||||
"code": "pending_bootstrap",
|
||||
"label": "待执行接入",
|
||||
"next_step": {
|
||||
"code": "execute_bootstrap_plan",
|
||||
"label": "去目标机执行接入片段",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"status_label": "待补执行面",
|
||||
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"route_surface": {"surface_complete": True, "missing_keys": []},
|
||||
"repository_capabilities": {},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
|
||||
self.assertEqual(
|
||||
"mainland-worker-01",
|
||||
payload["diagnosis"]["launchpad_recommended_target_node_code"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"签发接入工单",
|
||||
payload["diagnosis"]["launchpad_recommended_recovery_label"],
|
||||
)
|
||||
self.assertIn(
|
||||
"onboarding.bootstrap",
|
||||
payload["diagnosis"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "发布门禁待处理。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {
|
||||
"default_rollout_gate": {
|
||||
"status": "blocked",
|
||||
"summary_text": "还有节点未满足发布门禁。",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待处理",
|
||||
"recommended_action_code": "review_smart_rollout_preview",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
},
|
||||
"latest_release": {"release_version": "release-20260418"},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("release", decision["lane"])
|
||||
self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
|
||||
self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
|
||||
self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"release-launchpad http://127.0.0.1:8100 worker",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "发布门禁待处理。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {
|
||||
"default_rollout_gate": {
|
||||
"status": "attention",
|
||||
"summary_text": "当前还有节点未完成接入。",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"recommended_target_node_code": "mainland-worker-02",
|
||||
"recommended_recovery_label": "待执行接入",
|
||||
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"onboarding_bootstrap_pending_nodes": 1,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
},
|
||||
"latest_release": {"release_version": "release-20260418"},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("node_handover", decision["lane"])
|
||||
self.assertEqual("bootstrap_run", decision["primary_command_key"])
|
||||
self.assertEqual("focus-release-hub", decision["secondary_command_key"])
|
||||
self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
decision["summary"],
|
||||
)
|
||||
self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
|
||||
self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_includes_first_handover_gap_details(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "存在接管缺口。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "ssh_ready",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
"status_counts": {"pending_bootstrap": 1},
|
||||
"remote_access_state_counts": {"ssh_ready": 1},
|
||||
"delivery_queue_state_counts": {"healthy": 1},
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "已签发有效 Token,但目标节点尚未执行 bootstrap 片段。",
|
||||
"stage": {
|
||||
"code": "pending_bootstrap",
|
||||
"label": "待执行接入",
|
||||
"next_step": {
|
||||
"code": "execute_bootstrap_plan",
|
||||
"label": "去目标机执行接入片段",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"focus_ref": {"kind": "release_launchpad"},
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual(
|
||||
"mainland-worker-01",
|
||||
payload["managed_nodes"]["first_handover_gap_node_code"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"pending_bootstrap",
|
||||
payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
|
||||
)
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
|
||||
)
|
||||
self.assertIn("mainland-worker-01", issue["detail"])
|
||||
self.assertTrue(
|
||||
any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
|
||||
)
|
||||
self.assertIn(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
|
||||
payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
|
||||
)
|
||||
mock_get_managed_node_handover.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "pbr-worker-logs-001",
|
||||
"status": "attention",
|
||||
"status_label": "待关注",
|
||||
"focus_step_key": "worker_logs",
|
||||
"focus_summary": "Worker 日志步骤仍未收口。",
|
||||
"focus_ref": {
|
||||
"kind": "playbook_run",
|
||||
"run_code": "pbr-worker-logs-001",
|
||||
"focus_step_key": "worker_logs",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_ops_activity_stream.return_value = {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"status": "running",
|
||||
"summary_text": "远端 Agent 已回传关键日志样本。",
|
||||
"occurred_at": "2026-04-18 09:22:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 71,
|
||||
"event_key": "job-event:71",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 71,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
|
||||
self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
|
||||
top_item = payload["activity_stream"]["top_items"][0]
|
||||
self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
|
||||
self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job_events", top_item["ui_intent_kind"])
|
||||
self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "参与节点样本尚未形成。",
|
||||
"next_actions": [
|
||||
{
|
||||
"label": "看参与节点日志",
|
||||
"action_code": "open_worker_logs_participating",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"payload": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "busy",
|
||||
}
|
||||
],
|
||||
"participation_summary": {
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 1,
|
||||
"missing_participating_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
|
||||
)
|
||||
self.assertEqual("open_worker_logs_participating", issue["action_code"])
|
||||
self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
|
||||
self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
|
||||
self.assertTrue(
|
||||
any(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
|
||||
for command in issue["commands"]
|
||||
)
|
||||
)
|
||||
self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
|
||||
self.assertTrue(
|
||||
any(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
|
||||
for command in payload["diagnosis"]["quick_commands"]
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
any(
|
||||
"driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
|
||||
for command in issue["commands"]
|
||||
)
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "最近有任务回执异常。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"job_status": "running",
|
||||
"summary_text": "节点已接单执行,但开始回执未成功送达控制面。",
|
||||
"occurred_at": "2026-04-18 10:22:00",
|
||||
"job_code": "ops-agent-81",
|
||||
"start_delivery_state": "failed_local",
|
||||
"start_delivery_error": "temporary offline",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 81,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
|
||||
)
|
||||
self.assertEqual("focus_latest_job_events", issue["action_code"])
|
||||
self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
|
||||
self.assertIn("temporary offline", issue["detail"])
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("ops_jobs", decision["lane"])
|
||||
self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
|
||||
self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
177
domain-api/tests/test_ops_template_service.py
Normal file
177
domain-api/tests/test_ops_template_service.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
|
||||
|
||||
|
||||
class OpsTemplateServiceTests(unittest.TestCase):
|
||||
def test_node_bootstrap_defaults_and_clamps_number(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"node.bootstrap",
|
||||
{
|
||||
"expires_in_hours": 9999,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("ok", message)
|
||||
self.assertEqual("", data["control_plane_base_url"])
|
||||
self.assertEqual("/opt/domaincheck", data["root_dir"])
|
||||
self.assertEqual(720, data["expires_in_hours"])
|
||||
self.assertEqual("", data["node_region"])
|
||||
self.assertEqual("", data["node_role"])
|
||||
|
||||
def test_deploy_release_custom_normalizes_lists_and_booleans(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.custom",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
"restart_services": "domaincheck-api\ndomaincheck-worker",
|
||||
"health_check_urls": "",
|
||||
"health_check_services": "",
|
||||
"rollback_on_failure": "false",
|
||||
"switch_current": "true",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("ok", message)
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], data["restart_services"])
|
||||
self.assertEqual([], data["health_check_urls"])
|
||||
self.assertEqual([], data["health_check_services"])
|
||||
self.assertFalse(data["rollback_on_failure"])
|
||||
self.assertTrue(data["switch_current"])
|
||||
self.assertEqual("/opt/domaincheck", data["install_root"])
|
||||
|
||||
def test_deploy_release_control_applies_defaults(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.control",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
data["restart_services"],
|
||||
)
|
||||
self.assertEqual(["http://127.0.0.1:8100/health"], data["health_check_urls"])
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
data["health_check_services"],
|
||||
)
|
||||
self.assertTrue(data["rollback_on_failure"])
|
||||
self.assertTrue(data["switch_current"])
|
||||
|
||||
def test_deploy_release_missing_required_field_fails(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.custom",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("artifact_url 不能为空", message)
|
||||
self.assertEqual({}, data)
|
||||
|
||||
def test_logs_collect_lines_are_clamped(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 9999,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual(500, data["lines"])
|
||||
|
||||
def test_delivery_queue_templates_normalize_and_require_expected_fields(self) -> None:
|
||||
replay_ok, replay_message, replay_data = build_ops_template_payload(
|
||||
"delivery.queue.replay",
|
||||
{
|
||||
"record_id": "evt-001",
|
||||
"limit": 999,
|
||||
"flush_after_replay": "false",
|
||||
"reason": "确认状态已修复",
|
||||
},
|
||||
)
|
||||
discard_ok, discard_message, discard_data = build_ops_template_payload(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"record_id": "evt-002",
|
||||
"limit": 0,
|
||||
"reason": "确认这条死信不再需要补发",
|
||||
},
|
||||
)
|
||||
flush_ok, flush_message, flush_data = build_ops_template_payload(
|
||||
"delivery.queue.flush",
|
||||
{
|
||||
"limit": 999,
|
||||
},
|
||||
)
|
||||
discard_fail_ok, discard_fail_message, discard_fail_data = build_ops_template_payload(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"record_id": "evt-003",
|
||||
"reason": "",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(replay_ok, replay_message)
|
||||
self.assertEqual(200, replay_data["limit"])
|
||||
self.assertFalse(replay_data["flush_after_replay"])
|
||||
self.assertEqual("evt-001", replay_data["record_id"])
|
||||
|
||||
self.assertTrue(discard_ok, discard_message)
|
||||
self.assertEqual(1, discard_data["limit"])
|
||||
self.assertEqual("确认这条死信不再需要补发", discard_data["reason"])
|
||||
|
||||
self.assertTrue(flush_ok, flush_message)
|
||||
self.assertEqual(200, flush_data["limit"])
|
||||
|
||||
self.assertFalse(discard_fail_ok)
|
||||
self.assertEqual("reason 不能为空", discard_fail_message)
|
||||
self.assertEqual({}, discard_fail_data)
|
||||
|
||||
def test_diagnostics_and_service_control_templates_expose_ssh(self) -> None:
|
||||
health_template = get_ops_action_template("health.snapshot")
|
||||
logs_template = get_ops_action_template("logs.collect")
|
||||
diagnostics_template = get_ops_action_template("diagnostics.collect")
|
||||
service_status_template = get_ops_action_template("service.status")
|
||||
restart_template = get_ops_action_template("service.restart")
|
||||
restart_api_template = get_ops_action_template("runtime.restart_api")
|
||||
stop_worker_template = get_ops_action_template("runtime.stop_worker")
|
||||
deploy_control_template = get_ops_action_template("deploy.release.control")
|
||||
deploy_worker_template = get_ops_action_template("deploy.release.worker")
|
||||
deploy_custom_template = get_ops_action_template("deploy.release.custom")
|
||||
queue_flush_template = get_ops_action_template("delivery.queue.flush")
|
||||
queue_replay_template = get_ops_action_template("delivery.queue.replay")
|
||||
queue_discard_template = get_ops_action_template("delivery.queue.discard")
|
||||
|
||||
self.assertIn("ssh", health_template["execution_modes"])
|
||||
self.assertEqual("远端 Agent", health_template["default_execution_mode_label"])
|
||||
self.assertEqual("SSH", next(item["label"] for item in health_template["execution_mode_options"] if item["value"] == "ssh"))
|
||||
self.assertIn("ssh", logs_template["execution_modes"])
|
||||
self.assertIn("ssh", diagnostics_template["execution_modes"])
|
||||
self.assertIn("ssh", service_status_template["execution_modes"])
|
||||
self.assertIn("ssh", restart_template["execution_modes"])
|
||||
self.assertIn("ssh", restart_api_template["execution_modes"])
|
||||
self.assertIn("ssh", stop_worker_template["execution_modes"])
|
||||
self.assertEqual("控制面", get_ops_action_template("node.bootstrap")["default_execution_mode_label"])
|
||||
self.assertIn("ssh", deploy_control_template["execution_modes"])
|
||||
self.assertIn("ssh", deploy_worker_template["execution_modes"])
|
||||
self.assertIn("ssh", deploy_custom_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_flush_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_replay_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_discard_template["execution_modes"])
|
||||
self.assertEqual("远端 Agent", queue_flush_template["default_execution_mode_label"])
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
117
domain-api/tests/test_prepare_final_release.py
Normal file
117
domain-api/tests/test_prepare_final_release.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class PrepareFinalReleaseTests(unittest.TestCase):
|
||||
def test_prepare_final_release_blocks_when_smoke_is_false_even_if_verify_passes(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
source_script = repo_root / "prepare_final_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
release_root = root / "release"
|
||||
release_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
(root / "package_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"mkdir -p \"$PWD/release\"\n"
|
||||
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
|
||||
"{\n"
|
||||
" \"package_name\": \"domaincheck_release_test\",\n"
|
||||
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
|
||||
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
|
||||
" \"sha256\": \"abc\",\n"
|
||||
" \"smoke_test_ok\": false,\n"
|
||||
" \"smoke_test_message\": \"skipped by env\"\n"
|
||||
"}\n"
|
||||
"EOF\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "verify_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(root / "prepare_final_release.sh", 0o755)
|
||||
os.chmod(root / "package_domain_release.sh", 0o755)
|
||||
os.chmod(root / "verify_domain_release.sh", 0o755)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(root / "prepare_final_release.sh")],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
|
||||
self.assertFalse(payload["ok"])
|
||||
self.assertEqual("blocked", payload["release_gate"]["decision"])
|
||||
self.assertTrue(payload["release_gate"]["verify_ok"])
|
||||
self.assertFalse(payload["release_gate"]["smoke_test_ok"])
|
||||
self.assertIn("smoke_test_failed_or_skipped", payload["release_gate"]["blocked_reasons"])
|
||||
self.assertIn('"ok": false', result.stdout)
|
||||
|
||||
def test_prepare_final_release_ready_when_verify_and_smoke_both_pass(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
source_script = repo_root / "prepare_final_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
release_root = root / "release"
|
||||
release_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
(root / "package_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"mkdir -p \"$PWD/release\"\n"
|
||||
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
|
||||
"{\n"
|
||||
" \"package_name\": \"domaincheck_release_test\",\n"
|
||||
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
|
||||
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
|
||||
" \"sha256\": \"abc\",\n"
|
||||
" \"smoke_test_ok\": true,\n"
|
||||
" \"smoke_test_message\": \"all checks passed (1/1)\"\n"
|
||||
"}\n"
|
||||
"EOF\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "verify_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(root / "prepare_final_release.sh", 0o755)
|
||||
os.chmod(root / "package_domain_release.sh", 0o755)
|
||||
os.chmod(root / "verify_domain_release.sh", 0o755)
|
||||
|
||||
subprocess.run(
|
||||
["bash", str(root / "prepare_final_release.sh")],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
|
||||
self.assertTrue(payload["ok"])
|
||||
self.assertEqual("ready", payload["release_gate"]["decision"])
|
||||
self.assertEqual([], payload["release_gate"]["blocked_reasons"])
|
||||
self.assertTrue(payload["release_gate"]["verify_ok"])
|
||||
self.assertTrue(payload["release_gate"]["smoke_test_ok"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
29
domain-api/tests/test_release_metadata_parity.py
Normal file
29
domain-api/tests/test_release_metadata_parity.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ReleaseMetadataParityTests(unittest.TestCase):
|
||||
def test_powershell_release_package_emits_archive_path_alias(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_text = (repo_root / "package_domain_release.ps1").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("archive_path = $zipPath", script_text)
|
||||
self.assertIn('"archive_path=$zipPath"', script_text)
|
||||
|
||||
def test_powershell_release_flow_prefers_archive_path_with_zip_fallback(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
prepare_text = (repo_root / "prepare_final_release.ps1").read_text(encoding="utf-8")
|
||||
show_text = (repo_root / "show_latest_release.ps1").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("$archivePath = if ($latest.archive_path) { $latest.archive_path } else { $latest.zip_path }", prepare_text)
|
||||
self.assertIn('& (Join-Path $root "package_domain_release.ps1")', prepare_text)
|
||||
self.assertIn('& (Join-Path $root "verify_domain_release.ps1")', prepare_text)
|
||||
self.assertNotIn('powershell -ExecutionPolicy Bypass -File (Join-Path $root "package_domain_release.ps1")', prepare_text)
|
||||
self.assertIn("$reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path }", show_text)
|
||||
self.assertIn("$latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path }", show_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
19
domain-api/tests/test_release_script_parity.py
Normal file
19
domain-api/tests/test_release_script_parity.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ReleaseScriptParityTests(unittest.TestCase):
|
||||
def test_shell_release_verifier_checks_cross_platform_delivery_scripts(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "verify_domain_release.sh"
|
||||
script_text = script_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn('"scripts/package_domain_release.ps1"', script_text)
|
||||
self.assertIn('"scripts/verify_domain_release.ps1"', script_text)
|
||||
self.assertIn('"scripts/smoke_test_stack.ps1"', script_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
domain-api/tests/test_show_latest_release.py
Normal file
162
domain-api/tests/test_show_latest_release.py
Normal file
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ShowLatestReleaseTests(unittest.TestCase):
|
||||
def test_show_latest_release_marks_stale_final_report_as_not_ok(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_new",
|
||||
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
|
||||
"sha256": "new-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": "domaincheck_release_old",
|
||||
"archive_path": str(release_root / "domaincheck_release_old.tar.gz"),
|
||||
"sha256": "old-sha256",
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertFalse(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertFalse(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_old", payload["final_release_report_package_name"])
|
||||
self.assertIn("package_name", payload["final_release_report_stale_reason"])
|
||||
self.assertEqual("", payload["final_release_gate_decision"])
|
||||
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
|
||||
|
||||
def test_show_latest_release_keeps_final_report_ok_when_it_matches_latest(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_new",
|
||||
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
|
||||
"sha256": "new-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": latest_payload,
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertTrue(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertTrue(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_new", payload["final_release_report_package_name"])
|
||||
self.assertEqual("", payload["final_release_report_stale_reason"])
|
||||
self.assertEqual("ready", payload["final_release_gate_decision"])
|
||||
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
|
||||
self.assertTrue(payload["final_release_gate_verify_ok"])
|
||||
self.assertTrue(payload["final_release_gate_smoke_test_ok"])
|
||||
|
||||
def test_show_latest_release_accepts_zip_path_fallback(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_win",
|
||||
"zip_path": str(release_root / "domaincheck_release_win.zip"),
|
||||
"sha256": "zip-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": latest_payload,
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertTrue(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertTrue(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_win", payload["final_release_report_package_name"])
|
||||
self.assertEqual("ready", payload["final_release_gate_decision"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user